ddeutil-workflow 0.0.33__py3-none-any.whl → 0.0.35__py3-none-any.whl

ddeutil/workflow/{stage.py → stages.py}

@@ -42,9 +42,9 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
+from .caller import TagFunc, extract_call
 from .conf import config, get_logger
 from .exceptions import StageException
-from .hook import TagFunc, extract_hook
 from .result import Result, Status
 from .templates import not_in_template, param2template
 from .utils import (
@@ -60,7 +60,7 @@ __all__: TupleStr = (
     "EmptyStage",
     "BashStage",
     "PyStage",
-    "HookStage",
+    "CallStage",
     "TriggerStage",
     "Stage",
 )
@@ -100,7 +100,7 @@ class BaseStage(BaseModel, ABC):
         return self.id or self.name
 
     @model_validator(mode="after")
-    def __prepare_running_id__(self) -> Self:
+    def __prepare_running_id(self) -> Self:
         """Prepare stage running ID that use default value of field and this
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
@@ -140,24 +140,26 @@ class BaseStage(BaseModel, ABC):
         params: DictData,
         *,
         run_id: str | None = None,
+        parent_run_id: str | None = None,
         result: Result | None = None,
     ) -> Result:
-        """Handler result from the stage execution.
+        """Handler execution result from the stage `execute` method.
 
             This stage exception handler still use ok-error concept, but it
         allows you force catching an output result with error message by
         specific environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
 
             Execution   --> Ok      --> Result
-                                        |-status: 0
+                                        |-status: Status.SUCCESS
                                         |-context:
                                             |-outputs: ...
 
                         --> Error   --> Result (if env var was set)
-                                        |-status: 1
-                                        |-context:
-                                            |-error: ...
-                                            |-error_message: ...
+                                        |-status: Status.FAILED
+                                        |-errors:
+                                            |-class: ...
+                                            |-name: ...
+                                            |-message: ...
 
                         --> Error   --> Raise StageException(...)
 
@@ -166,32 +168,31 @@ class BaseStage(BaseModel, ABC):
 
         :param params: A parameter data that want to use in this execution.
         :param run_id: (str) A running stage ID for this execution.
+        :param parent_run_id: A parent workflow running ID for this release.
         :param result: (Result) A result object for keeping context and status
             data.
 
         :rtype: Result
         """
-        if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(
-                    run_id or gen_id(self.name + (self.id or ""), unique=True)
-                ),
-            )
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=(self.name + (self.id or "")),
+        )
 
         try:
-            # NOTE: Start calling origin function with a passing args.
             return self.execute(params, result=result)
         except Exception as err:
-            # NOTE: Start catching error from the stage execution.
             result.trace.error(f"[STAGE]: {err.__class__.__name__}: {err}")
+
             if config.stage_raise_error:
                 # NOTE: If error that raise from stage execution course by
                 #   itself, it will return that error with previous
                 #   dependency.
                 if isinstance(err, StageException):
-                    raise StageException(
-                        f"{self.__class__.__name__}: \n\t{err}"
-                    ) from err
+                    raise
+
                 raise StageException(
                     f"{self.__class__.__name__}: \n\t"
                     f"{err.__class__.__name__}: {err}"
@@ -202,8 +203,11 @@ class BaseStage(BaseModel, ABC):
             return result.catch(
                 status=Status.FAILED,
                 context={
-                    "error": err,
-                    "error_message": f"{err.__class__.__name__}: {err}",
+                    "errors": {
+                        "class": err,
+                        "name": err.__class__.__name__,
+                        "message": f"{err.__class__.__name__}: {err}",
+                    },
                 },
             )
 
@@ -247,8 +251,12 @@ class BaseStage(BaseModel, ABC):
             else gen_id(param2template(self.name, params=to))
         )
 
+        errors: DictData = (
+            {"errors": output.pop("errors", {})} if "errors" in output else {}
+        )
+
         # NOTE: Set the output to that stage generated ID with ``outputs`` key.
-        to["stages"][_id] = {"outputs": output}
+        to["stages"][_id] = {"outputs": output, **errors}
         return to
 
     def is_skipped(self, params: DictData | None = None) -> bool:
@@ -321,6 +329,11 @@ class EmptyStage(BaseStage):
 
         :rtype: Result
         """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         result.trace.info(
             f"[STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
@@ -413,6 +426,11 @@ class BashStage(BaseStage):
 
         :rtype: Result
         """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         bash: str = param2template(dedent(self.bash), params)
 
         result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
@@ -473,12 +491,24 @@ class PyStage(BaseStage):
     )
 
     @staticmethod
-    def pick_keys_from_locals(values: DictData) -> Iterator[str]:
-        from inspect import ismodule
+    def filter_locals(values: DictData) -> Iterator[str]:
+        """Filter a locals input values.
+
+        :param values: (DictData) A locals values that want to filter.
+
+        :rtype: Iterator[str]
+        """
+        from inspect import isclass, ismodule
 
         for value in values:
-            if value == "__annotations__" or ismodule(values[value]):
+
+            if (
+                value == "__annotations__"
+                or ismodule(values[value])
+                or isclass(values[value])
+            ):
                 continue
+
             yield value
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
@@ -494,7 +524,7 @@ class PyStage(BaseStage):
         lc: DictData = output.get("locals", {})
         super().set_outputs(
             (
-                {k: lc[k] for k in self.pick_keys_from_locals(lc)}
+                {k: lc[k] for k in self.filter_locals(lc)}
                 | {k: output[k] for k in output if k.startswith("error")}
             ),
             to=to,
@@ -518,6 +548,11 @@ class PyStage(BaseStage):
 
         :rtype: Result
         """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         # NOTE: Replace the run statement that has templating value.
         run: str = param2template(dedent(self.run), params)
 
@@ -539,8 +574,8 @@ class PyStage(BaseStage):
         )
 
 
-class HookStage(BaseStage):
-    """Hook executor that hook the Python function from registry with tag
+class CallStage(BaseStage):
+    """Call executor that call the Python function from registry with tag
     decorator function in ``utils`` module and run it with input arguments.
 
         This stage is different with PyStage because the PyStage is just calling
@@ -558,23 +593,23 @@ class HookStage(BaseStage):
 
     uses: str = Field(
         description=(
-            "A pointer that want to load function from the hook registry."
+            "A pointer that want to load function from the call registry."
         ),
     )
     args: DictData = Field(
         default_factory=dict,
-        description="An arguments that want to pass to the hook function.",
+        description="An arguments that want to pass to the call function.",
         alias="with",
     )
 
     def execute(
         self, params: DictData, *, result: Result | None = None
     ) -> Result:
-        """Execute the Hook function that already in the hook registry.
+        """Execute the Call function that already in the call registry.
 
-        :raise ValueError: When the necessary arguments of hook function do not
+        :raise ValueError: When the necessary arguments of call function do not
             set from the input params argument.
-        :raise TypeError: When the return type of hook function does not be
+        :raise TypeError: When the return type of call function does not be
             dict type.
 
         :param params: A parameter that want to pass before run any statement.
@@ -585,7 +620,12 @@ class HookStage(BaseStage):
 
         :rtype: Result
         """
-        t_func: TagFunc = extract_hook(param2template(self.uses, params))()
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
+        t_func: TagFunc = extract_call(param2template(self.uses, params))()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
@@ -608,11 +648,11 @@ class HookStage(BaseStage):
         if "result" not in ips.parameters:
             args.pop("result")
 
-        result.trace.info(f"[STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}")
+        result.trace.info(f"[STAGE]: Call-Execute: {t_func.name}@{t_func.tag}")
         rs: DictData = t_func(**param2template(args, params))
 
         # VALIDATE:
-        #   Check the result type from hook function, it should be dict.
+        #   Check the result type from call function, it should be dict.
         if not isinstance(rs, dict):
             raise TypeError(
                 f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
@@ -659,14 +699,19 @@ class TriggerStage(BaseStage):
         # NOTE: Lazy import this workflow object.
         from . import Workflow
 
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
         # NOTE: Loading workflow object from trigger name.
         _trigger: str = param2template(self.trigger, params=params)
 
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
-        wf: Workflow = Workflow.from_loader(name=_trigger)
+        workflow: Workflow = Workflow.from_loader(name=_trigger)
         result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
-        return wf.execute(
+        return workflow.execute(
             params=param2template(self.params, params),
             result=result,
         )
@@ -680,19 +725,37 @@ class TriggerStage(BaseStage):
 Stage = Union[
     PyStage,
     BashStage,
-    HookStage,
+    CallStage,
     TriggerStage,
     EmptyStage,
 ]
 
 
 # TODO: Not implement this stages yet
-class ParallelStage(BaseModel):  # pragma: no cov
+class ParallelStage(BaseStage):  # pragma: no cov
     parallel: list[Stage]
     max_parallel_core: int = Field(default=2)
 
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...
+
+
+# TODO: Not implement this stages yet
+class ForEachStage(BaseStage):  # pragma: no cov
+    foreach: list[str]
+    stages: list[Stage]
+
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...
+
 
 # TODO: Not implement this stages yet
-class ForEachStage(BaseModel):  # pragma: no cov
+class HookStage(BaseStage):  # pragma: no cov
     foreach: list[str]
     stages: list[Stage]
+
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...

ddeutil/workflow/utils.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import logging
 import stat
 import time
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
 from datetime import datetime, timedelta
 from hashlib import md5
 from inspect import isfunction
@@ -199,7 +199,7 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
     )
 
 
-def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
+def batch(iterable: Iterator[Any] | range, n: int) -> Iterator[Any]:
     """Batch data into iterators of length n. The last batch may be shorter.
 
     Example:
@@ -240,3 +240,21 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
     :rtype: str
     """
     return run_id[-num:]
+
+
+def deep_update(origin: DictData, u: Mapping) -> DictData:
+    """Deep update dict.
+
+    Example:
+        >>> deep_update(
+        ...     origin={"jobs": {"job01": "foo"}},
+        ...     u={"jobs": {"job02": "bar"}},
+        ... )
+        {"jobs": {"job01": "foo", "job02": "bar"}}
+    """
+    for k, value in u.items():
+        if isinstance(value, Mapping) and value:
+            origin[k] = deep_update(origin.get(k, {}), value)
+        else:
+            origin[k] = value
+    return origin