ddeutil-workflow 0.0.32__py3-none-any.whl → 0.0.34__py3-none-any.whl

ddeutil/workflow/{stage.py → stages.py}

@@ -42,10 +42,10 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
+from .call import TagFunc, extract_call
 from .conf import config, get_logger
 from .exceptions import StageException
-from .hook import TagFunc, extract_hook
-from .result import Result
+from .result import Result, Status
 from .templates import not_in_template, param2template
 from .utils import (
     cut_id,
@@ -60,7 +60,7 @@ __all__: TupleStr = (
     "EmptyStage",
     "BashStage",
     "PyStage",
-    "HookStage",
+    "CallStage",
     "TriggerStage",
     "Stage",
 )
@@ -100,7 +100,7 @@ class BaseStage(BaseModel, ABC):
         return self.id or self.name
 
     @model_validator(mode="after")
-    def __prepare_running_id__(self) -> Self:
+    def __prepare_running_id(self) -> Self:
         """Prepare stage running ID that use default value of field and this
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
@@ -121,36 +121,45 @@ class BaseStage(BaseModel, ABC):
         return self
 
     @abstractmethod
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
     def handler_execute(
-        self, params: DictData, *, run_id: str | None = None
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        result: Result | None = None,
     ) -> Result:
-        """Handler result from the stage execution.
+        """Handler execution result from the stage `execute` method.
 
             This stage exception handler still use ok-error concept, but it
         allows you force catching an output result with error message by
         specific environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
 
             Execution   --> Ok      --> Result
-                                        |-status: 0
+                                        |-status: Status.SUCCESS
                                         |-context:
                                             |-outputs: ...
 
                         --> Error   --> Result (if env var was set)
-                                        |-status: 1
-                                        |-context:
-                                            |-error: ...
-                                            |-error_message: ...
+                                        |-status: Status.FAILED
+                                        |-errors:
+                                            |-class: ...
+                                            |-name: ...
+                                            |-message: ...
 
                         --> Error   --> Raise StageException(...)
 
@@ -158,31 +167,35 @@ class BaseStage(BaseModel, ABC):
         from current stage ID before release the final result.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param run_id: (str) A running stage ID for this execution.
+        :param parent_run_id: A parent workflow running ID for this release.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        if not run_id:
-            run_id: str = gen_id(self.name + (self.id or ""), unique=True)
+        if result is None:
+            result: Result = Result(
+                run_id=(
+                    run_id or gen_id(self.name + (self.id or ""), unique=True)
+                ),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
 
-        rs_raise: Result = Result(status=1, run_id=run_id)
         try:
-            # NOTE: Start calling origin function with a passing args.
-            return self.execute(params, run_id=run_id)
+            return self.execute(params, result=result)
         except Exception as err:
-            # NOTE: Start catching error from the stage execution.
-            logger.error(
-                f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
-                f"{err}"
-            )
+            result.trace.error(f"[STAGE]: {err.__class__.__name__}: {err}")
+
             if config.stage_raise_error:
                 # NOTE: If error that raise from stage execution course by
                 #   itself, it will return that error with previous
                 #   dependency.
                 if isinstance(err, StageException):
-                    raise StageException(
-                        f"{self.__class__.__name__}: \n\t{err}"
-                    ) from err
+                    raise
+
                 raise StageException(
                     f"{self.__class__.__name__}: \n\t"
                     f"{err.__class__.__name__}: {err}"
@@ -190,11 +203,14 @@ class BaseStage(BaseModel, ABC):
 
             # NOTE: Catching exception error object to result with
             #   error_message and error keys.
-            return rs_raise.catch(
-                status=1,
+            return result.catch(
+                status=Status.FAILED,
                 context={
-                    "error": err,
-                    "error_message": f"{err.__class__.__name__}: {err}",
+                    "errors": {
+                        "class": err,
+                        "name": err.__class__.__name__,
+                        "message": f"{err.__class__.__name__}: {err}",
+                    },
                 },
             )
 
@@ -238,8 +254,12 @@ class BaseStage(BaseModel, ABC):
             else gen_id(param2template(self.name, params=to))
         )
 
+        errors: DictData = (
+            {"errors": output.pop("errors", {})} if "errors" in output else {}
+        )
+
         # NOTE: Set the output to that stage generated ID with ``outputs`` key.
-        to["stages"][_id] = {"outputs": output}
+        to["stages"][_id] = {"outputs": output, **errors}
         return to
 
     def is_skipped(self, params: DictData | None = None) -> bool:
@@ -295,7 +315,9 @@ class EmptyStage(BaseStage):
         ge=0,
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
         it does not get any error from logging function.
@@ -305,22 +327,26 @@ class EmptyStage(BaseStage):
 
         :param params: A context data that want to add output result. But this
             stage does not pass any output.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
+        result.trace.info(
+            f"[STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
             if self.sleep > 30:
-                logger.info(
-                    f"({cut_id(run_id)}) [STAGE]: ... sleep "
-                    f"({self.sleep} seconds)"
-                )
+                result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
             time.sleep(self.sleep)
-        return Result(status=0, context={}, run_id=run_id)
+
+        return result.catch(status=Status.SUCCESS)
 
 
 class BashStage(BaseStage):
@@ -334,7 +360,7 @@ class BashStage(BaseStage):
 
     Data Validate:
         >>> stage = {
-        ...     "name": "Shell stage execution",
+        ...     "name": "The Shell stage execution",
         ...     "bash": 'echo "Hello $FOO"',
         ...     "env": {
         ...         "FOO": "BAR",
@@ -391,20 +417,30 @@ class BashStage(BaseStage):
         # Note: Remove .sh file that use to run bash.
         Path(f"./{f_name}").unlink()
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
+        result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
-            bash=bash, env=param2template(self.env, params), run_id=run_id
+            bash=bash,
+            env=param2template(self.env, params),
+            run_id=result.run_id,
         ) as sh:
             rs: CompletedProcess = subprocess.run(
                 sh, shell=False, capture_output=True, text=True
@@ -420,14 +456,13 @@ class BashStage(BaseStage):
                 f"Subprocess: {err}\nRunning Statement:\n---\n"
                 f"```bash\n{bash}\n```"
             )
-        return Result(
-            status=0,
+        return result.catch(
+            status=Status.SUCCESS,
             context={
                 "return_code": rs.returncode,
                 "stdout": rs.stdout.rstrip("\n") or None,
                 "stderr": rs.stderr.rstrip("\n") or None,
             },
-            run_id=run_id,
         )
 
 
@@ -459,12 +494,24 @@ class PyStage(BaseStage):
     )
 
     @staticmethod
-    def pick_keys_from_locals(values: DictData) -> Iterator[str]:
-        from inspect import ismodule
+    def filter_locals(values: DictData) -> Iterator[str]:
+        """Filter a locals input values.
+
+        :param values: (DictData) A locals values that want to filter.
+
+        :rtype: Iterator[str]
+        """
+        from inspect import isclass, ismodule
 
         for value in values:
-            if value == "__annotations__" or ismodule(values[value]):
+
+            if (
+                value == "__annotations__"
+                or ismodule(values[value])
+                or isclass(values[value])
+            ):
                 continue
+
             yield value
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
@@ -480,7 +527,7 @@ class PyStage(BaseStage):
         lc: DictData = output.get("locals", {})
         super().set_outputs(
             (
-                {k: lc[k] for k in self.pick_keys_from_locals(lc)}
+                {k: lc[k] for k in self.filter_locals(lc)}
                 | {k: output[k] for k in output if k.startswith("error")}
             ),
             to=to,
@@ -492,15 +539,23 @@ class PyStage(BaseStage):
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
 
         :param params: A parameter that want to pass before run any statement.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         # NOTE: Replace the run statement that has templating value.
         run: str = param2template(dedent(self.run), params)
 
@@ -511,21 +566,19 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
+        result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
         exec(run, _globals, lc)
 
-        return Result(
-            status=0,
-            context={"locals": lc, "globals": _globals},
-            run_id=run_id,
+        return result.catch(
+            status=Status.SUCCESS, context={"locals": lc, "globals": _globals}
         )
 
 
-class HookStage(BaseStage):
-    """Hook executor that hook the Python function from registry with tag
+class CallStage(BaseStage):
+    """Call executor that call the Python function from registry with tag
     decorator function in ``utils`` module and run it with input arguments.
 
         This stage is different with PyStage because the PyStage is just calling
@@ -543,35 +596,43 @@ class HookStage(BaseStage):
 
     uses: str = Field(
         description=(
-            "A pointer that want to load function from the hook registry."
+            "A pointer that want to load function from the call registry."
         ),
     )
     args: DictData = Field(
         default_factory=dict,
-        description="An arguments that want to pass to the hook function.",
+        description="An arguments that want to pass to the call function.",
         alias="with",
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
-        """Execute the Hook function that already in the hook registry.
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
+        """Execute the Call function that already in the call registry.
 
-        :raise ValueError: When the necessary arguments of hook function do not
+        :raise ValueError: When the necessary arguments of call function do not
             set from the input params argument.
-        :raise TypeError: When the return type of hook function does not be
+        :raise TypeError: When the return type of call function does not be
             dict type.
 
         :param params: A parameter that want to pass before run any statement.
         :type params: DictData
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :type: str | None
 
         :rtype: Result
         """
-        t_func: TagFunc = extract_hook(param2template(self.uses, params))()
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
+        t_func: TagFunc = extract_call(param2template(self.uses, params))()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = param2template(self.args, params)
+        args: DictData = {"result": result} | param2template(self.args, params)
         ips = inspect.signature(t_func)
         if any(
             (k.removeprefix("_") not in args and k not in args)
@@ -587,20 +648,20 @@ class HookStage(BaseStage):
             if k.removeprefix("_") in args:
                 args[k] = args.pop(k.removeprefix("_"))
 
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
-            f"{t_func.name}@{t_func.tag}"
-        )
+        if "result" not in ips.parameters:
+            args.pop("result")
+
+        result.trace.info(f"[STAGE]: Call-Execute: {t_func.name}@{t_func.tag}")
         rs: DictData = t_func(**param2template(args, params))
 
         # VALIDATE:
-        #   Check the result type from hook function, it should be dict.
+        #   Check the result type from call function, it should be dict.
         if not isinstance(rs, dict):
             raise TypeError(
                 f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
                 f"to result model, you change return type to `dict`."
             )
-        return Result(status=0, context=rs, run_id=run_id)
+        return result.catch(status=Status.SUCCESS, context=rs)
 
 
 class TriggerStage(BaseStage):
@@ -626,31 +687,37 @@ class TriggerStage(BaseStage):
         description="A parameter that want to pass to workflow execution.",
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
         # NOTE: Lazy import this workflow object.
         from . import Workflow
 
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         # NOTE: Loading workflow object from trigger name.
         _trigger: str = param2template(self.trigger, params=params)
 
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
-        wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
-        )
-        return wf.execute(
+        workflow: Workflow = Workflow.from_loader(name=_trigger)
+        result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
+        return workflow.execute(
             params=param2template(self.params, params),
-            run_id=run_id,
-        ).set_run_id(run_id)
+            result=result,
+        )
 
 
 # NOTE:
@@ -661,19 +728,37 @@ class TriggerStage(BaseStage):
 Stage = Union[
     PyStage,
     BashStage,
-    HookStage,
+    CallStage,
     TriggerStage,
     EmptyStage,
 ]
 
 
 # TODO: Not implement this stages yet
-class ParallelStage(BaseModel):  # pragma: no cov
+class ParallelStage(BaseStage):  # pragma: no cov
     parallel: list[Stage]
     max_parallel_core: int = Field(default=2)
 
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...
+
+
+# TODO: Not implement this stages yet
+class ForEachStage(BaseStage):  # pragma: no cov
+    foreach: list[str]
+    stages: list[Stage]
+
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...
+
 
 # TODO: Not implement this stages yet
-class ForEachStage(BaseModel):  # pragma: no cov
+class HookStage(BaseStage):  # pragma: no cov
     foreach: list[str]
     stages: list[Stage]
+
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...

ddeutil/workflow/utils.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import logging
 import stat
 import time
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
 from datetime import datetime, timedelta
 from hashlib import md5
 from inspect import isfunction
@@ -199,7 +199,7 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
     )
 
 
-def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
+def batch(iterable: Iterator[Any] | range, n: int) -> Iterator[Any]:
     """Batch data into iterators of length n. The last batch may be shorter.
 
     Example:
@@ -240,3 +240,21 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
     :rtype: str
     """
     return run_id[-num:]
+
+
+def deep_update(origin: DictData, u: Mapping) -> DictData:
+    """Deep update dict.
+
+    Example:
+        >>> deep_update(
+        ...     origin={"jobs": {"job01": "foo"}},
+        ...     u={"jobs": {"job02": "bar"}},
+        ... )
+        {"jobs": {"job01": "foo", "job02": "bar"}}
+    """
+    for k, value in u.items():
+        if isinstance(value, Mapping) and value:
+            origin[k] = deep_update(origin.get(k, {}), value)
+        else:
+            origin[k] = value
+    return origin