ddeutil-workflow 0.0.41__py3-none-any.whl → 0.0.43__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -3,19 +3,19 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use config
-"""Stage Model that use for getting stage data template from the Job Model.
-The stage handle the minimize task that run in some thread (same thread at
-its job owner) that mean it is the lowest executor of a workflow that can
-tracking logs.
+# [x] Use dynamic config
+"""Stage model. It stores all stage model that use for getting stage data template
+from the Job Model. The stage handle the minimize task that run in some thread
+(same thread at its job owner) that mean it is the lowest executor of a workflow
+that can tracking logs.
 
     The output of stage execution only return 0 status because I do not want to
 handle stage error on this stage model. I think stage model should have a lot of
 use-case, and it does not worry when I want to create a new one.
 
-    Execution   --> Ok      --> Result with 0
+    Execution   --> Ok      --> Result with SUCCESS
 
-                --> Error   ┬-> Result with 1 (if env var was set)
+                --> Error   ┬-> Result with FAILED (if env var was set)
                             ╰-> Raise StageException(...)
 
     On the context I/O that pass to a stage object at execute process. The
@@ -50,9 +50,9 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
-from .conf import config
+from .conf import dynamic
 from .exceptions import StageException, to_dict
-from .result import Result, Status
+from .result import FAILED, SUCCESS, Result, Status
 from .reusables import TagFunc, extract_call, not_in_template, param2template
 from .utils import (
     gen_id,
@@ -67,6 +67,7 @@ __all__: TupleStr = (
     "TriggerStage",
     "ForEachStage",
     "ParallelStage",
+    "RaiseStage",
     "Stage",
 )
 
@@ -79,6 +80,10 @@ class BaseStage(BaseModel, ABC):
         This class is the abstraction class for any stage class.
     """
 
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra override config values.",
+    )
     id: Optional[str] = Field(
         default=None,
         description=(
@@ -94,10 +99,6 @@ class BaseStage(BaseModel, ABC):
         description="A stage condition statement to allow stage executable.",
         alias="if",
     )
-    extras: DictData = Field(
-        default_factory=dict,
-        description="An extra override values.",
-    )
 
     @property
     def iden(self) -> str:
@@ -151,15 +152,6 @@ class BaseStage(BaseModel, ABC):
         """
         raise NotImplementedError("Stage should implement `execute` method.")
 
-    async def axecute(
-        self,
-        params: DictData,
-        *,
-        result: Result | None = None,
-        event: Event | None,
-    ) -> Result:  # pragma: no cov
-        ...
-
     def handler_execute(
         self,
         params: DictData,
@@ -167,7 +159,7 @@ class BaseStage(BaseModel, ABC):
         run_id: str | None = None,
         parent_run_id: str | None = None,
         result: Result | None = None,
-        raise_error: bool = False,
+        raise_error: bool | None = None,
         to: DictData | None = None,
         event: Event | None = None,
     ) -> Result:
@@ -178,12 +170,12 @@ class BaseStage(BaseModel, ABC):
         specific environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
 
             Execution   --> Ok      --> Result
-                                        |-status: Status.SUCCESS
+                                        |-status: SUCCESS
                                         ╰-context:
                                             ╰-outputs: ...
 
                         --> Error   --> Result (if env var was set)
-                                        |-status: Status.FAILED
+                                        |-status: FAILED
                                         ╰-errors:
                                             |-class: ...
                                             |-name: ...
@@ -217,26 +209,24 @@ class BaseStage(BaseModel, ABC):
 
         try:
             rs: Result = self.execute(params, result=result, event=event)
-            if to is not None:
-                return self.set_outputs(rs.context, to=to)
-            return rs
-        except Exception as err:
-            result.trace.error(f"[STAGE]: {err.__class__.__name__}: {err}")
+            return self.set_outputs(rs.context, to=to) if to is not None else rs
+        except Exception as e:
+            result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
 
-            if raise_error or config.stage_raise_error:
-                if isinstance(err, StageException):
+            if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
+                if isinstance(e, StageException):
                     raise
 
                 raise StageException(
                     f"{self.__class__.__name__}: \n\t"
-                    f"{err.__class__.__name__}: {err}"
-                ) from None
+                    f"{e.__class__.__name__}: {e}"
+                ) from e
 
-            errors: DictData = {"errors": to_dict(err)}
+            errors: DictData = {"errors": to_dict(e)}
             if to is not None:
                 return self.set_outputs(errors, to=to)
 
-            return result.catch(status=Status.FAILED, context=errors)
+            return result.catch(status=FAILED, context=errors)
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
@@ -268,13 +258,17 @@ class BaseStage(BaseModel, ABC):
         if "stages" not in to:
             to["stages"] = {}
 
-        if self.id is None and not config.stage_default_id:
+        if self.id is None and not dynamic(
+            "stage_default_id", extras=self.extras
+        ):
             return to
 
         _id: str = (
-            param2template(self.id, params=to)
+            param2template(self.id, params=to, extras=self.extras)
             if self.id
-            else gen_id(param2template(self.name, params=to))
+            else gen_id(
+                param2template(self.name, params=to, extras=self.extras)
+            )
         )
 
         errors: DictData = (
@@ -312,16 +306,111 @@ class BaseStage(BaseModel, ABC):
             #   should use the `re` module to validate eval-string before
             #   running.
             rs: bool = eval(
-                param2template(self.condition, params), globals() | params, {}
+                param2template(self.condition, params, extras=self.extras),
+                globals() | params,
+                {},
             )
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
             return not rs
-        except Exception as err:
-            raise StageException(f"{err.__class__.__name__}: {err}") from err
+        except Exception as e:
+            raise StageException(f"{e.__class__.__name__}: {e}") from e
+
+
+class BaseAsyncStage(BaseStage):
+
+    @abstractmethod
+    def execute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
+    ) -> Result: ...
+
+    @abstractmethod
+    async def axecute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
+    ) -> Result:
+        """Async execution method for this Empty stage that only logging out to
+        stdout.
+
+        :param params: (DictData) A context data that want to add output result.
+            But this stage does not pass any output.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        raise NotImplementedError(
+            "Async Stage should implement `axecute` method."
+        )
+
+    async def handler_axecute(
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        result: Result | None = None,
+        raise_error: bool | None = None,
+        to: DictData | None = None,
+        event: Event | None = None,
+    ) -> Result:
+        """Async Handler stage execution result from the stage `execute` method.
+
+        :param params: (DictData) A parameterize value data that use in this
+            stage execution.
+        :param run_id: (str) A running stage ID for this execution.
+        :param parent_run_id: (str) A parent workflow running ID for this
+            execution.
+        :param result: (Result) A result object for keeping context and status
+            data before execution.
+        :param raise_error: (bool) A flag that all this method raise error
+        :param to: (DictData) A target object for auto set the return output
+            after execution.
+        :param event: (Event) An event manager that pass to the stage execution.
+
+        :rtype: Result
+        """
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=self.iden,
+        )
+
+        try:
+            rs: Result = await self.axecute(params, result=result, event=event)
+            if to is not None:
+                return self.set_outputs(rs.context, to=to)
+            return rs
+        except Exception as e:
+            await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
+
+            if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
+                if isinstance(e, StageException):
+                    raise
+
+                raise StageException(
+                    f"{self.__class__.__name__}: \n\t"
+                    f"{e.__class__.__name__}: {e}"
+                ) from None
+
+            errors: DictData = {"errors": to_dict(e)}
+            if to is not None:
+                return self.set_outputs(errors, to=to)
 
+            return result.catch(status=FAILED, context=errors)
 
-class EmptyStage(BaseStage):
+
+class EmptyStage(BaseAsyncStage):
     """Empty stage that do nothing (context equal empty stage) and logging the
     name of stage only to stdout.
 
@@ -372,23 +461,22 @@ class EmptyStage(BaseStage):
 
         result.trace.info(
             f"[STAGE]: Empty-Execute: {self.name!r}: "
-            f"( {param2template(self.echo, params=params) or '...'} )"
+            f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
         )
         if self.sleep > 0:
             if self.sleep > 5:
                 result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
             time.sleep(self.sleep)
 
-        return result.catch(status=Status.SUCCESS)
+        return result.catch(status=SUCCESS)
 
-    # TODO: Draft async execute method for the perf improvement.
     async def axecute(
         self,
         params: DictData,
         *,
         result: Result | None = None,
-        event: Event | None,
-    ) -> Result:  # pragma: no cov
+        event: Event | None = None,
+    ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
 
@@ -406,17 +494,19 @@ class EmptyStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        result.trace.info(
+        await result.trace.ainfo(
             f"[STAGE]: Empty-Execute: {self.name!r}: "
-            f"( {param2template(self.echo, params=params) or '...'} )"
+            f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
         )
 
         if self.sleep > 0:
             if self.sleep > 5:
-                result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
+                await result.trace.ainfo(
+                    f"[STAGE]: ... sleep ({self.sleep} seconds)"
+                )
             await asyncio.sleep(self.sleep)
 
-        return result.catch(status=Status.SUCCESS)
+        return result.catch(status=SUCCESS)
 
 
 class BashStage(BaseStage):
@@ -508,12 +598,14 @@ class BashStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        bash: str = param2template(dedent(self.bash), params)
+        bash: str = param2template(
+            dedent(self.bash), params, extras=self.extras
+        )
 
         result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
             bash=bash,
-            env=param2template(self.env, params),
+            env=param2template(self.env, params, extras=self.extras),
             run_id=result.run_id,
         ) as sh:
             result.trace.debug(f"... Start create `{sh[1]}` file.")
@@ -523,17 +615,17 @@ class BashStage(BaseStage):
 
         if rs.returncode > 0:
             # NOTE: Prepare stderr message that returning from subprocess.
-            err: str = (
+            e: str = (
                 rs.stderr.encode("utf-8").decode("utf-16")
                 if "\\x00" in rs.stderr
                 else rs.stderr
             ).removesuffix("\n")
             raise StageException(
-                f"Subprocess: {err}\nRunning Statement:\n---\n"
+                f"Subprocess: {e}\nRunning Statement:\n---\n"
                 f"```bash\n{bash}\n```"
             )
         return result.catch(
-            status=Status.SUCCESS,
+            status=SUCCESS,
             context={
                 "return_code": rs.returncode,
                 "stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
@@ -635,7 +727,7 @@ class PyStage(BaseStage):
         gb: DictData = (
             globals()
             | params
-            | param2template(self.vars, params)
+            | param2template(self.vars, params, extras=self.extras)
             | {"result": result}
         )
 
@@ -648,10 +740,12 @@ class PyStage(BaseStage):
 
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
-        exec(param2template(dedent(self.run), params), gb, lc)
+        exec(
+            param2template(dedent(self.run), params, extras=self.extras), gb, lc
+        )
 
         return result.catch(
-            status=Status.SUCCESS, context={"locals": lc, "globals": gb}
+            status=SUCCESS, context={"locals": lc, "globals": gb}
         )
 
 
@@ -716,11 +810,16 @@ class CallStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        t_func: TagFunc = extract_call(param2template(self.uses, params))()
+        t_func: TagFunc = extract_call(
+            param2template(self.uses, params, extras=self.extras),
+            registries=self.extras.get("regis_call"),
+        )()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = {"result": result} | param2template(self.args, params)
+        args: DictData = {"result": result} | param2template(
+            self.args, params, extras=self.extras
+        )
         ips = inspect.signature(t_func)
         necessary_params: list[str] = [
             k
@@ -754,10 +853,12 @@ class CallStage(BaseStage):
         if inspect.iscoroutinefunction(t_func):  # pragma: no cov
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
-                t_func(**param2template(args, params))
+                t_func(**param2template(args, params, extras=self.extras))
             )
         else:
-            rs: DictData = t_func(**param2template(args, params))
+            rs: DictData = t_func(
+                **param2template(args, params, extras=self.extras)
+            )
 
         # VALIDATE:
         #   Check the result type from call function, it should be dict.
@@ -766,7 +867,7 @@ class CallStage(BaseStage):
                 f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
                 f"to result model, you change return type to `dict`."
             )
-        return result.catch(status=Status.SUCCESS, context=rs)
+        return result.catch(status=SUCCESS, context=rs)
 
 
 class TriggerStage(BaseStage):
@@ -819,15 +920,16 @@ class TriggerStage(BaseStage):
             )
 
         # NOTE: Loading workflow object from trigger name.
-        _trigger: str = param2template(self.trigger, params=params)
+        _trigger: str = param2template(self.trigger, params, extras=self.extras)
 
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
-        workflow: Workflow = Workflow.from_loader(name=_trigger)
+        workflow: Workflow = Workflow.from_conf(_trigger, extras=self.extras)
         result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
         return workflow.execute(
-            params=param2template(self.params, params),
+            params=param2template(self.params, params, extras=self.extras),
             result=result,
+            event=event,
         )
 
 
@@ -893,19 +995,11 @@ class ParallelStage(BaseStage):  # pragma: no cov
                     ).context,
                     to=context,
                 )
-            except StageException as err:  # pragma: no cov
+            except StageException as e:  # pragma: no cov
                 result.trace.error(
-                    f"[STAGE]: Catch:\n\t{err.__class__.__name__}:" f"\n\t{err}"
-                )
-                context.update(
-                    {
-                        "errors": {
-                            "class": err,
-                            "name": err.__class__.__name__,
-                            "message": f"{err.__class__.__name__}: {err}",
-                        },
-                    },
+                    f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
                 )
+                context.update({"errors": e.to_dict()})
         return context
 
     def execute(
@@ -931,8 +1025,11 @@ class ParallelStage(BaseStage):  # pragma: no cov
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
+        result.trace.info(
+            f"[STAGE]: Parallel-Execute with {self.max_parallel_core} cores."
+        )
         rs: DictData = {"parallel": {}}
-        status = Status.SUCCESS
+        status = SUCCESS
         with ThreadPoolExecutor(
             max_workers=self.max_parallel_core,
             thread_name_prefix="parallel_stage_exec_",
@@ -956,7 +1053,7 @@ class ParallelStage(BaseStage):  # pragma: no cov
                 rs["parallel"][context.pop("branch")] = context
 
                 if "errors" in context:
-                    status = Status.FAILED
+                    status = FAILED
 
         return result.catch(status=status, context=rs)
 
@@ -981,7 +1078,7 @@ class ForEachStage(BaseStage):
         ... }
     """
 
-    foreach: Union[list[str], list[int]] = Field(
+    foreach: Union[list[str], list[int], str] = Field(
         description=(
             "A items for passing to each stages via ${{ item }} template."
         ),
@@ -1023,10 +1120,21 @@ class ForEachStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        rs: DictData = {"items": self.foreach, "foreach": {}}
-        status = Status.SUCCESS
+        foreach: Union[list[str], list[int]] = (
+            param2template(self.foreach, params, extras=self.extras)
+            if isinstance(self.foreach, str)
+            else self.foreach
+        )
+        if not isinstance(foreach, list):
+            raise StageException(
+                f"Foreach does not support foreach value: {foreach!r}"
+            )
+
+        result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
+        rs: DictData = {"items": foreach, "foreach": {}}
+        status: Status = SUCCESS
         # TODO: Implement concurrent more than 1.
-        for item in self.foreach:
+        for item in foreach:
             result.trace.debug(f"[STAGE]: Execute foreach item: {item!r}")
             params["item"] = item
             context = {"stages": {}}
@@ -1041,21 +1149,12 @@ class ForEachStage(BaseStage):
                         ).context,
                         to=context,
                     )
-                except StageException as err:  # pragma: no cov
-                    status = Status.FAILED
+                except StageException as e:  # pragma: no cov
+                    status = FAILED
                     result.trace.error(
-                        f"[STAGE]: Catch:\n\t{err.__class__.__name__}:"
-                        f"\n\t{err}"
-                    )
-                    context.update(
-                        {
-                            "errors": {
-                                "class": err,
-                                "name": err.__class__.__name__,
-                                "message": f"{err.__class__.__name__}: {err}",
-                            },
-                        },
+                        f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
                     )
+                    context.update({"errors": e.to_dict()})
 
             rs["foreach"][item] = context
 
@@ -1064,10 +1163,24 @@ class ForEachStage(BaseStage):
 
 # TODO: Not implement this stages yet
 class UntilStage(BaseStage):  # pragma: no cov
-    """Until execution stage."""
+    """Until execution stage.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "Until stage execution",
+        ...     "item": 1,
+        ...     "until": "${{ item }} > 3"
+        ...     "stages": [
+        ...         {
+        ...             "name": "Start increase item value.",
+        ...             "run": "item = ${{ item }}\\nitem += 1\\n"
+        ...         },
+        ...     ],
+        ... }
+    """
 
-    until: str = Field(description="A until condition.")
     item: Union[str, int, bool] = Field(description="An initial value.")
+    until: str = Field(description="A until condition.")
     stages: list[Stage] = Field(
         default_factory=list,
         description=(
@@ -1090,8 +1203,14 @@ class UntilStage(BaseStage):  # pragma: no cov
 
 
 # TODO: Not implement this stages yet
-class IfStage(BaseStage):  # pragma: no cov
-    """If execution stage.
+class Match(BaseModel):
+    case: Union[str, int]
+    stage: Stage
+
+
+# TODO: Not implement this stages yet
+class CaseStage(BaseStage):  # pragma: no cov
+    """Case execution stage.
 
     Data Validate:
         >>> stage = {
@@ -1125,7 +1244,7 @@ class IfStage(BaseStage):  # pragma: no cov
     """
 
     case: str = Field(description="A case condition for routing.")
-    match: list[dict[str, Union[str, Stage]]]
+    match: list[Match]
 
     def execute(
         self,
@@ -1133,7 +1252,51 @@ class IfStage(BaseStage):  # pragma: no cov
         *,
         result: Result | None = None,
         event: Event | None = None,
-    ) -> Result: ...
+    ) -> Result:
+        """Execute case-match condition that pass to the case field.
+
+        :param params: A parameter that want to pass before run any statement.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+        status = SUCCESS
+        _case = param2template(self.case, params, extras=self.extras)
+        _else = None
+        context = {}
+        for match in self.match:
+            if (c := match.case) != "_":
+                _condition = param2template(c, params, extras=self.extras)
+            else:
+                _else = match
+                continue
+
+            if match == _condition:
+                stage: Stage = match.stage
+                try:
+                    stage.set_outputs(
+                        stage.handler_execute(
+                            params=params,
+                            run_id=result.run_id,
+                            parent_run_id=result.parent_run_id,
+                        ).context,
+                        to=context,
+                    )
+                except StageException as e:  # pragma: no cov
+                    status = FAILED
+                    result.trace.error(
+                        f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
+                    )
+                    context.update({"errors": e.to_dict()})
+
+        return result.catch(status=status, context=context)
 
 
 class RaiseStage(BaseStage):  # pragma: no cov
@@ -1165,13 +1328,14 @@ class RaiseStage(BaseStage):  # pragma: no cov
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
-
-        result.trace.error(f"[STAGE]: ... raise ( {self.message} )")
+        result.trace.info(f"[STAGE]: Raise-Execute: {self.message!r}.")
         raise StageException(self.message)
 
 
 # TODO: Not implement this stages yet
 class HookStage(BaseStage):  # pragma: no cov
+    """Hook stage execution."""
+
     hook: str
     args: DictData
     callback: str
@@ -1187,7 +1351,20 @@ class HookStage(BaseStage):  # pragma: no cov
 
 # TODO: Not implement this stages yet
 class DockerStage(BaseStage):  # pragma: no cov
-    """Docker container stage execution."""
+    """Docker container stage execution.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "Docker stage execution",
+        ...     "image": "image-name.pkg.com",
+        ...     "env": {
+        ...         "ENV": "dev",
+        ...     },
+        ...     "volume": {
+        ...         "secrets": "/secrets",
+        ...     },
+        ... }
+    """
 
     image: str = Field(
         description="A Docker image url with tag that want to run.",
@@ -1224,18 +1401,6 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         return super().execute(params, result=result)
 
 
-# TODO: Not implement this stages yet
-class SensorStage(BaseStage):  # pragma: no cov
-
-    def execute(
-        self,
-        params: DictData,
-        *,
-        result: Result | None = None,
-        event: Event | None = None,
-    ) -> Result: ...
-
-
 # NOTE:
 #   An order of parsing stage model on the Job model with ``stages`` field.
 #   From the current build-in stages, they do not have stage that have the same
@@ -1243,7 +1408,6 @@ class SensorStage(BaseStage):  # pragma: no cov
 #
 Stage = Annotated[
     Union[
-        EmptyStage,
         BashStage,
         CallStage,
         TriggerStage,
@@ -1251,6 +1415,7 @@ Stage = Annotated[
         ParallelStage,
         PyStage,
         RaiseStage,
+        EmptyStage,
     ],
     Field(union_mode="smart"),
 ]