ddeutil-workflow 0.0.41__py3-none-any.whl → 0.0.42__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use config
+# [x] Use dynamic config
 """Stage Model that use for getting stage data template from the Job Model.
 The stage handle the minimize task that run in some thread (same thread at
 its job owner) that mean it is the lowest executor of a workflow that can
@@ -50,7 +50,7 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
-from .conf import config
+from .conf import dynamic
 from .exceptions import StageException, to_dict
 from .result import Result, Status
 from .reusables import TagFunc, extract_call, not_in_template, param2template
@@ -67,6 +67,7 @@ __all__: TupleStr = (
     "TriggerStage",
     "ForEachStage",
     "ParallelStage",
+    "RaiseStage",
     "Stage",
 )
 
@@ -96,7 +97,7 @@ class BaseStage(BaseModel, ABC):
     )
     extras: DictData = Field(
         default_factory=dict,
-        description="An extra override values.",
+        description="An extra override config values.",
     )
 
     @property
@@ -151,15 +152,6 @@ class BaseStage(BaseModel, ABC):
         """
         raise NotImplementedError("Stage should implement `execute` method.")
 
-    async def axecute(
-        self,
-        params: DictData,
-        *,
-        result: Result | None = None,
-        event: Event | None,
-    ) -> Result:  # pragma: no cov
-        ...
-
     def handler_execute(
         self,
         params: DictData,
@@ -167,7 +159,7 @@ class BaseStage(BaseModel, ABC):
         run_id: str | None = None,
         parent_run_id: str | None = None,
         result: Result | None = None,
-        raise_error: bool = False,
+        raise_error: bool | None = None,
         to: DictData | None = None,
         event: Event | None = None,
     ) -> Result:
@@ -223,7 +215,7 @@ class BaseStage(BaseModel, ABC):
         except Exception as err:
             result.trace.error(f"[STAGE]: {err.__class__.__name__}: {err}")
 
-            if raise_error or config.stage_raise_error:
+            if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
                 if isinstance(err, StageException):
                     raise
 
@@ -268,13 +260,17 @@ class BaseStage(BaseModel, ABC):
         if "stages" not in to:
             to["stages"] = {}
 
-        if self.id is None and not config.stage_default_id:
+        if self.id is None and not dynamic(
+            "stage_default_id", extras=self.extras
+        ):
             return to
 
         _id: str = (
-            param2template(self.id, params=to)
+            param2template(self.id, params=to, extras=self.extras)
             if self.id
-            else gen_id(param2template(self.name, params=to))
+            else gen_id(
+                param2template(self.name, params=to, extras=self.extras)
+            )
         )
 
         errors: DictData = (
@@ -312,7 +308,9 @@ class BaseStage(BaseModel, ABC):
             #   should use the `re` module to validate eval-string before
             #   running.
             rs: bool = eval(
-                param2template(self.condition, params), globals() | params, {}
+                param2template(self.condition, params, extras=self.extras),
+                globals() | params,
+                {},
             )
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
@@ -321,7 +319,102 @@ class BaseStage(BaseModel, ABC):
             raise StageException(f"{err.__class__.__name__}: {err}") from err
 
 
-class EmptyStage(BaseStage):
+class BaseAsyncStage(BaseStage):
+
+    @abstractmethod
+    def execute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
+    ) -> Result: ...
+
+    @abstractmethod
+    async def axecute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
+    ) -> Result:
+        """Async execution method for this Empty stage that only logging out to
+        stdout.
+
+        :param params: (DictData) A context data that want to add output result.
+            But this stage does not pass any output.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        raise NotImplementedError(
+            "Async Stage should implement `axecute` method."
+        )
+
+    async def handler_axecute(
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        result: Result | None = None,
+        raise_error: bool | None = None,
+        to: DictData | None = None,
+        event: Event | None = None,
+    ) -> Result:
+        """Async Handler stage execution result from the stage `execute` method.
+
+        :param params: (DictData) A parameterize value data that use in this
+            stage execution.
+        :param run_id: (str) A running stage ID for this execution.
+        :param parent_run_id: (str) A parent workflow running ID for this
+            execution.
+        :param result: (Result) A result object for keeping context and status
+            data before execution.
+        :param raise_error: (bool) A flag that all this method raise error
+        :param to: (DictData) A target object for auto set the return output
+            after execution.
+        :param event: (Event) An event manager that pass to the stage execution.
+
+        :rtype: Result
+        """
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=self.iden,
+        )
+
+        try:
+            rs: Result = await self.axecute(params, result=result, event=event)
+            if to is not None:
+                return self.set_outputs(rs.context, to=to)
+            return rs
+        except Exception as err:
+            await result.trace.aerror(
+                f"[STAGE]: {err.__class__.__name__}: {err}"
+            )
+
+            if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
+                if isinstance(err, StageException):
+                    raise
+
+                raise StageException(
+                    f"{self.__class__.__name__}: \n\t"
+                    f"{err.__class__.__name__}: {err}"
+                ) from None
+
+            errors: DictData = {"errors": to_dict(err)}
+            if to is not None:
+                return self.set_outputs(errors, to=to)
+
+            return result.catch(status=Status.FAILED, context=errors)
+
+
+class EmptyStage(BaseAsyncStage):
     """Empty stage that do nothing (context equal empty stage) and logging the
     name of stage only to stdout.
 
@@ -372,7 +465,7 @@ class EmptyStage(BaseStage):
 
         result.trace.info(
             f"[STAGE]: Empty-Execute: {self.name!r}: "
-            f"( {param2template(self.echo, params=params) or '...'} )"
+            f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
         )
         if self.sleep > 0:
             if self.sleep > 5:
@@ -381,14 +474,13 @@ class EmptyStage(BaseStage):
 
         return result.catch(status=Status.SUCCESS)
 
-    # TODO: Draft async execute method for the perf improvement.
     async def axecute(
         self,
         params: DictData,
         *,
         result: Result | None = None,
-        event: Event | None,
-    ) -> Result:  # pragma: no cov
+        event: Event | None = None,
+    ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
 
@@ -406,14 +498,16 @@ class EmptyStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        result.trace.info(
+        await result.trace.ainfo(
             f"[STAGE]: Empty-Execute: {self.name!r}: "
-            f"( {param2template(self.echo, params=params) or '...'} )"
+            f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
         )
 
         if self.sleep > 0:
             if self.sleep > 5:
-                result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
+                await result.trace.ainfo(
+                    f"[STAGE]: ... sleep ({self.sleep} seconds)"
+                )
             await asyncio.sleep(self.sleep)
 
         return result.catch(status=Status.SUCCESS)
@@ -508,12 +602,14 @@ class BashStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        bash: str = param2template(dedent(self.bash), params)
+        bash: str = param2template(
+            dedent(self.bash), params, extras=self.extras
+        )
 
         result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
             bash=bash,
-            env=param2template(self.env, params),
+            env=param2template(self.env, params, extras=self.extras),
             run_id=result.run_id,
         ) as sh:
             result.trace.debug(f"... Start create `{sh[1]}` file.")
@@ -635,7 +731,7 @@ class PyStage(BaseStage):
         gb: DictData = (
             globals()
             | params
-            | param2template(self.vars, params)
+            | param2template(self.vars, params, extras=self.extras)
             | {"result": result}
         )
 
@@ -648,7 +744,9 @@ class PyStage(BaseStage):
 
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
-        exec(param2template(dedent(self.run), params), gb, lc)
+        exec(
+            param2template(dedent(self.run), params, extras=self.extras), gb, lc
+        )
 
         return result.catch(
             status=Status.SUCCESS, context={"locals": lc, "globals": gb}
@@ -716,11 +814,16 @@ class CallStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        t_func: TagFunc = extract_call(param2template(self.uses, params))()
+        t_func: TagFunc = extract_call(
+            param2template(self.uses, params, extras=self.extras),
+            registries=self.extras.get("regis_call"),
+        )()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = {"result": result} | param2template(self.args, params)
+        args: DictData = {"result": result} | param2template(
+            self.args, params, extras=self.extras
+        )
         ips = inspect.signature(t_func)
         necessary_params: list[str] = [
             k
@@ -754,10 +857,12 @@ class CallStage(BaseStage):
         if inspect.iscoroutinefunction(t_func):  # pragma: no cov
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
-                t_func(**param2template(args, params))
+                t_func(**param2template(args, params, extras=self.extras))
             )
         else:
-            rs: DictData = t_func(**param2template(args, params))
+            rs: DictData = t_func(
+                **param2template(args, params, extras=self.extras)
+            )
 
         # VALIDATE:
         #   Check the result type from call function, it should be dict.
@@ -819,15 +924,16 @@ class TriggerStage(BaseStage):
             )
 
         # NOTE: Loading workflow object from trigger name.
-        _trigger: str = param2template(self.trigger, params=params)
+        _trigger: str = param2template(self.trigger, params, extras=self.extras)
 
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
-        workflow: Workflow = Workflow.from_loader(name=_trigger)
+        workflow: Workflow = Workflow.from_conf(_trigger, extras=self.extras)
         result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
         return workflow.execute(
-            params=param2template(self.params, params),
+            params=param2template(self.params, params, extras=self.extras),
             result=result,
+            event=event,
         )
 
 
@@ -931,6 +1037,9 @@ class ParallelStage(BaseStage):  # pragma: no cov
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
+        result.trace.info(
+            f"[STAGE]: Parallel-Execute with {self.max_parallel_core} cores."
+        )
         rs: DictData = {"parallel": {}}
         status = Status.SUCCESS
         with ThreadPoolExecutor(
@@ -981,7 +1090,7 @@ class ForEachStage(BaseStage):
         ... }
     """
 
-    foreach: Union[list[str], list[int]] = Field(
+    foreach: Union[list[str], list[int], str] = Field(
         description=(
             "A items for passing to each stages via ${{ item }} template."
         ),
@@ -1023,10 +1132,21 @@ class ForEachStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        rs: DictData = {"items": self.foreach, "foreach": {}}
+        foreach: Union[list[str], list[int]] = (
+            param2template(self.foreach, params, extras=self.extras)
+            if isinstance(self.foreach, str)
+            else self.foreach
+        )
+        if not isinstance(foreach, list):
+            raise StageException(
+                f"Foreach does not support foreach value: {foreach!r}"
+            )
+
+        result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
+        rs: DictData = {"items": foreach, "foreach": {}}
         status = Status.SUCCESS
         # TODO: Implement concurrent more than 1.
-        for item in self.foreach:
+        for item in foreach:
             result.trace.debug(f"[STAGE]: Execute foreach item: {item!r}")
             params["item"] = item
             context = {"stages": {}}
@@ -1064,10 +1184,24 @@ class ForEachStage(BaseStage):
 
 # TODO: Not implement this stages yet
 class UntilStage(BaseStage):  # pragma: no cov
-    """Until execution stage."""
+    """Until execution stage.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "Until stage execution",
+        ...     "item": 1,
+        ...     "until": "${{ item }} > 3"
+        ...     "stages": [
+        ...         {
+        ...             "name": "Start increase item value.",
+        ...             "run": "item = ${{ item }}\\nitem += 1\\n"
+        ...         },
+        ...     ],
+        ... }
+    """
 
-    until: str = Field(description="A until condition.")
     item: Union[str, int, bool] = Field(description="An initial value.")
+    until: str = Field(description="A until condition.")
     stages: list[Stage] = Field(
         default_factory=list,
         description=(
@@ -1090,8 +1224,14 @@ class UntilStage(BaseStage):  # pragma: no cov
 
 
 # TODO: Not implement this stages yet
-class IfStage(BaseStage):  # pragma: no cov
-    """If execution stage.
+class Match(BaseModel):
+    case: Union[str, int]
+    stage: Stage
+
+
+# TODO: Not implement this stages yet
+class CaseStage(BaseStage):  # pragma: no cov
+    """Case execution stage.
 
     Data Validate:
         >>> stage = {
@@ -1125,7 +1265,7 @@ class IfStage(BaseStage):  # pragma: no cov
     """
 
     case: str = Field(description="A case condition for routing.")
-    match: list[dict[str, Union[str, Stage]]]
+    match: list[Match]
 
     def execute(
         self,
@@ -1133,7 +1273,60 @@ class IfStage(BaseStage):  # pragma: no cov
         *,
         result: Result | None = None,
         event: Event | None = None,
-    ) -> Result: ...
+    ) -> Result:
+        """Execute case-match condition that pass to the case field.
+
+        :param params: A parameter that want to pass before run any statement.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+        status = Status.SUCCESS
+        _case = param2template(self.case, params, extras=self.extras)
+        _else = None
+        context = {}
+        for match in self.match:
+            if (c := match.case) != "_":
+                _condition = param2template(c, params, extras=self.extras)
+            else:
+                _else = match
+                continue
+
+            if match == _condition:
+                stage: Stage = match.stage
+                try:
+                    stage.set_outputs(
+                        stage.handler_execute(
+                            params=params,
+                            run_id=result.run_id,
+                            parent_run_id=result.parent_run_id,
+                        ).context,
+                        to=context,
+                    )
+                except StageException as err:  # pragma: no cov
+                    status = Status.FAILED
+                    result.trace.error(
+                        f"[STAGE]: Catch:\n\t{err.__class__.__name__}:"
+                        f"\n\t{err}"
+                    )
+                    context.update(
+                        {
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
+                        },
+                    )
+
+        return result.catch(status=status, context=context)
 
 
 class RaiseStage(BaseStage):  # pragma: no cov
@@ -1165,13 +1358,14 @@ class RaiseStage(BaseStage):  # pragma: no cov
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
-
-        result.trace.error(f"[STAGE]: ... raise ( {self.message} )")
+        result.trace.info(f"[STAGE]: Raise-Execute: {self.message!r}.")
         raise StageException(self.message)
 
 
 # TODO: Not implement this stages yet
 class HookStage(BaseStage):  # pragma: no cov
+    """Hook stage execution."""
+
     hook: str
     args: DictData
     callback: str
@@ -1187,7 +1381,20 @@ class HookStage(BaseStage):  # pragma: no cov
 
 # TODO: Not implement this stages yet
 class DockerStage(BaseStage):  # pragma: no cov
-    """Docker container stage execution."""
+    """Docker container stage execution.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "Docker stage execution",
+        ...     "image": "image-name.pkg.com",
+        ...     "env": {
+        ...         "ENV": "dev",
+        ...     },
+        ...     "volume": {
+        ...         "secrets": "/secrets",
+        ...     },
+        ... }
+    """
 
     image: str = Field(
         description="A Docker image url with tag that want to run.",
@@ -1224,18 +1431,6 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         return super().execute(params, result=result)
 
 
-# TODO: Not implement this stages yet
-class SensorStage(BaseStage):  # pragma: no cov
-
-    def execute(
-        self,
-        params: DictData,
-        *,
-        result: Result | None = None,
-        event: Event | None = None,
-    ) -> Result: ...
-
-
 # NOTE:
 #   An order of parsing stage model on the Job model with ``stages`` field.
 #   From the current build-in stages, they do not have stage that have the same
@@ -1243,7 +1438,6 @@ class SensorStage(BaseStage):  # pragma: no cov
 #
 Stage = Annotated[
     Union[
-        EmptyStage,
         BashStage,
         CallStage,
         TriggerStage,
@@ -1251,6 +1445,7 @@ Stage = Annotated[
         ParallelStage,
         PyStage,
         RaiseStage,
+        EmptyStage,
     ],
     Field(union_mode="smart"),
 ]

ddeutil/workflow/utils.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [ ] Use config
 """Utility function model."""
 from __future__ import annotations