ddeutil-workflow 0.0.49__py3-none-any.whl → 0.0.50__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -26,6 +26,7 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
+import copy
 import inspect
 import subprocess
 import sys
@@ -34,10 +35,13 @@ import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from concurrent.futures import (
+    FIRST_EXCEPTION,
     Future,
     ThreadPoolExecutor,
     as_completed,
+    wait,
 )
+from datetime import datetime
 from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
@@ -51,10 +55,11 @@ from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
 from .conf import dynamic
-from .exceptions import StageException, to_dict
-from .result import FAILED, SUCCESS, Result, Status
+from .exceptions import StageException, UtilException, to_dict
+from .result import FAILED, SUCCESS, WAIT, Result, Status
 from .reusables import TagFunc, extract_call, not_in_template, param2template
 from .utils import (
+    filter_func,
     gen_id,
     make_exec,
 )
@@ -150,9 +155,8 @@ class BaseStage(BaseModel, ABC):
         parent_run_id: str | None = None,
         result: Result | None = None,
         raise_error: bool | None = None,
-        to: DictData | None = None,
         event: Event | None = None,
-    ) -> Result:
+    ) -> Result | DictData:
         """Handler stage execution result from the stage `execute` method.
 
             This stage exception handler still use ok-error concept, but it
@@ -184,8 +188,6 @@ class BaseStage(BaseModel, ABC):
         :param result: (Result) A result object for keeping context and status
             data before execution.
         :param raise_error: (bool) A flag that all this method raise error
-        :param to: (DictData) A target object for auto set the return output
-            after execution.
         :param event: (Event) An event manager that pass to the stage execution.
 
         :rtype: Result
@@ -200,7 +202,7 @@ class BaseStage(BaseModel, ABC):
 
         try:
             rs: Result = self.execute(params, result=result, event=event)
-            return self.set_outputs(rs.context, to=to) if to is not None else rs
+            return rs
         except Exception as e:
             result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
 
@@ -214,11 +216,7 @@ class BaseStage(BaseModel, ABC):
                 ) from e
 
             errors: DictData = {"errors": to_dict(e)}
-            return (
-                self.set_outputs(errors, to=to)
-                if to is not None
-                else result.catch(status=FAILED, context=errors)
-            )
+            return result.catch(status=FAILED, context=errors)
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
@@ -275,7 +273,10 @@ class BaseStage(BaseModel, ABC):
         return to
 
     def get_outputs(self, outputs: DictData) -> DictData:
-        """Get the outputs from stages data.
+        """Get the outputs from stages data. It will get this stage ID from
+        the stage outputs mapping.
+
+        :param outputs: (DictData) A stage outputs that want to get by stage ID.
 
         :rtype: DictData
         """
@@ -374,7 +375,6 @@ class BaseAsyncStage(BaseStage):
         parent_run_id: str | None = None,
         result: Result | None = None,
         raise_error: bool | None = None,
-        to: DictData | None = None,
         event: Event | None = None,
     ) -> Result:
         """Async Handler stage execution result from the stage `execute` method.
@@ -387,8 +387,6 @@ class BaseAsyncStage(BaseStage):
         :param result: (Result) A result object for keeping context and status
             data before execution.
         :param raise_error: (bool) A flag that all this method raise error
-        :param to: (DictData) A target object for auto set the return output
-            after execution.
         :param event: (Event) An event manager that pass to the stage execution.
 
         :rtype: Result
@@ -403,8 +401,6 @@ class BaseAsyncStage(BaseStage):
 
         try:
             rs: Result = await self.axecute(params, result=result, event=event)
-            if to is not None:  # pragma: no cov
-                return self.set_outputs(rs.context, to=to)
             return rs
         except Exception as e:  # pragma: no cov
             await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
@@ -419,9 +415,6 @@ class BaseAsyncStage(BaseStage):
                 ) from None
 
             errors: DictData = {"errors": to_dict(e)}
-            if to is not None:
-                return self.set_outputs(errors, to=to)
-
             return result.catch(status=FAILED, context=errors)
 
 
@@ -880,7 +873,7 @@ class CallStage(BaseStage):
         # VALIDATE:
         #   Check the result type from call function, it should be dict.
         if isinstance(rs, BaseModel):
-            rs: DictData = rs.model_dump()
+            rs: DictData = rs.model_dump(by_alias=True)
         elif not isinstance(rs, dict):
             raise TypeError(
                 f"Return type: '{call_func.name}@{call_func.tag}' does not "
@@ -896,6 +889,10 @@ class CallStage(BaseStage):
     ) -> DictData:
         """Parse Pydantic model from any dict data before parsing to target
         caller function.
+
+        :param func:
+        :param args:
+        :param result: (Result)
         """
         try:
             type_hints: dict[str, Any] = get_type_hints(func)
@@ -942,7 +939,7 @@ class TriggerStage(BaseStage):
 
     trigger: str = Field(
         description=(
-            "A trigger workflow name that should already exist on the config."
+            "A trigger workflow name that should exist on the config path."
         ),
     )
     params: DictData = Field(
@@ -968,26 +965,30 @@ class TriggerStage(BaseStage):
 
         :rtype: Result
         """
-        # NOTE: Lazy import this workflow object.
-        from . import Workflow
+        from .workflow import Workflow
 
-        if result is None:  # pragma: no cov
+        if result is None:
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        # NOTE: Loading workflow object from trigger name.
         _trigger: str = param2template(self.trigger, params, extras=self.extras)
-
-        # NOTE: Set running workflow ID from running stage ID to external
-        #   params on Loader object.
-        workflow: Workflow = Workflow.from_conf(_trigger, extras=self.extras)
         result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
-        return workflow.execute(
+        rs: Result = Workflow.from_conf(
+            _trigger, extras=self.extras | {"stage_raise_error": True}
+        ).execute(
             params=param2template(self.params, params, extras=self.extras),
             result=result,
             event=event,
         )
+        if rs.status == FAILED:
+            err_msg: str | None = (
+                f" with:\n{msg}"
+                if (msg := rs.context.get("errors", {}).get("message"))
+                else ""
+            )
+            raise StageException(f"Trigger workflow was failed{err_msg}.")
+        return rs
 
 
 class ParallelStage(BaseStage):  # pragma: no cov
@@ -1124,10 +1125,10 @@ class ParallelStage(BaseStage):  # pragma: no cov
 
 class ForEachStage(BaseStage):
     """For-Each execution stage that execute child stages with an item in list
-    of item values.
+    of item values. This stage is not the low-level stage model because it runs
+    muti-stages in this stage execution.
 
-        This stage is not the low-level stage model because it runs muti-stages
-    in this stage execution.
+        The concept of this stage use the same logic of the Job execution.
 
     Data Validate:
         >>> stage = {
@@ -1167,46 +1168,91 @@ class ForEachStage(BaseStage):
         self,
         item: Union[str, int],
         params: DictData,
-        context: DictData,
         result: Result,
-    ) -> tuple[Status, DictData]:
+        *,
+        event: Event | None = None,
+    ) -> Result:
         """Execute foreach item from list of item.
 
         :param item: (str | int) An item that want to execution.
         :param params: (DictData) A parameter that want to pass to stage
             execution.
-        :param context: (DictData)
-        :param result: (Result)
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
-        :rtype: tuple[Status, DictData]
+        :rtype: Result
         """
-        result.trace.debug(f"[STAGE]: Execute foreach item: {item!r}")
-        params["item"] = item
-        to: DictData = {"item": item, "stages": {}}
-        status: Status = SUCCESS
+        result.trace.debug(f"[STAGE]: Execute item: {item!r}")
+        context: DictData = copy.deepcopy(params)
+        context.update({"item": item, "stages": {}})
         for stage in self.stages:
 
             if self.extras:
                 stage.extras = self.extras
 
-            try:
-                stage.set_outputs(
-                    stage.handler_execute(
-                        params=params,
-                        run_id=result.run_id,
-                        parent_run_id=result.parent_run_id,
-                    ).context,
-                    to=to,
+            if stage.is_skipped(params=context):
+                result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
+                stage.set_outputs(output={"skipped": True}, to=context)
+                continue
+
+            if event and event.is_set():
+                error_msg: str = (
+                    "Item-Stage was canceled from event that had set before "
+                    "stage item execution."
                 )
-            except StageException as e:  # pragma: no cov
-                status = FAILED
-                result.trace.error(
-                    f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
+                return result.catch(
+                    status=FAILED,
+                    foreach={
+                        item: {
+                            "item": item,
+                            "stages": filter_func(context.pop("stages", {})),
+                            "errors": StageException(error_msg).to_dict(),
+                        }
+                    },
+                )
+
+            try:
+                rs: Result = stage.handler_execute(
+                    params=context,
+                    run_id=result.run_id,
+                    parent_run_id=result.parent_run_id,
+                    raise_error=True,
                 )
-                to.update({"errors": e.to_dict()})
+                stage.set_outputs(rs.context, to=context)
+                if rs.status == FAILED:
+                    error_msg: str = (
+                        f"Item-Stage was break because it has a sub stage, "
+                        f"{stage.iden}, failed without raise error."
+                    )
+                    return result.catch(
+                        status=FAILED,
+                        foreach={
+                            item: {
+                                "item": item,
+                                "stages": filter_func(
+                                    context.pop("stages", {})
+                                ),
+                                "errors": StageException(error_msg).to_dict(),
+                            },
+                        },
+                    )
+            except (StageException, UtilException) as e:
+                result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
+                raise StageException(
+                    f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
+                ) from None
 
-        context["foreach"][item] = to
-        return status, context
+        return result.catch(
+            status=SUCCESS,
+            foreach={
+                item: {
+                    "item": item,
+                    "stages": filter_func(context.pop("stages", {})),
+                },
+            },
+        )
 
     def execute(
         self,
@@ -1227,9 +1273,10 @@ class ForEachStage(BaseStage):
         """
         if result is None:  # pragma: no cov
             result: Result = Result(
-                run_id=gen_id(self.name + (self.id or ""), unique=True)
+                run_id=gen_id(self.name + (self.id or ""), unique=True),
+                extras=self.extras,
             )
-
+        event: Event = Event() if event is None else event
         foreach: Union[list[str], list[int]] = (
             param2template(self.foreach, params, extras=self.extras)
             if isinstance(self.foreach, str)
@@ -1241,24 +1288,57 @@ class ForEachStage(BaseStage):
             )
 
         result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
-        context: DictData = {"items": foreach, "foreach": {}}
-        statuses: list[Union[SUCCESS, FAILED]] = []
-        with ThreadPoolExecutor(max_workers=self.concurrent) as executor:
+        result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
+        if event and event.is_set():  # pragma: no cov
+            return result.catch(
+                status=FAILED,
+                context={
+                    "errors": StageException(
+                        "Stage was canceled from event that had set "
+                        "before stage foreach execution."
+                    ).to_dict()
+                },
+            )
+
+        with ThreadPoolExecutor(
+            max_workers=self.concurrent, thread_name_prefix="stage_foreach_"
+        ) as executor:
+
             futures: list[Future] = [
                 executor.submit(
                     self.execute_item,
                     item=item,
-                    params=params.copy(),
-                    context=context,
+                    params=params,
                     result=result,
                 )
                 for item in foreach
             ]
-            for future in as_completed(futures):
-                status, context = future.result()
-                statuses.append(status)
+            context: DictData = {}
+            status: Status = SUCCESS
 
-        return result.catch(status=max(statuses), context=context)
+            done, not_done = wait(
+                futures, timeout=1800, return_when=FIRST_EXCEPTION
+            )
+
+            if len(done) != len(futures):
+                result.trace.warning(
+                    "[STAGE]: Set the event for stop running stage."
+                )
+                event.set()
+                for future in not_done:
+                    future.cancel()
+
+            for future in done:
+                try:
+                    future.result()
+                except StageException as e:
+                    status = FAILED
+                    result.trace.error(
+                        f"[STAGE]: Catch:\n\t{e.__class__.__name__}:\n\t{e}"
+                    )
+                    context.update({"errors": e.to_dict()})
+
+        return result.catch(status=status, context=context)
 
 
 class UntilStage(BaseStage):  # pragma: no cov
@@ -1299,9 +1379,9 @@ class UntilStage(BaseStage):  # pragma: no cov
         item: T,
         loop: int,
         params: DictData,
-        context: DictData,
         result: Result,
-    ) -> tuple[Status, DictData, T]:
+        event: Event | None = None,
+    ) -> tuple[Result, T]:
         """Execute until item set item by some stage or by default loop
         variable.
 
@@ -1309,43 +1389,80 @@ class UntilStage(BaseStage):  # pragma: no cov
         :param loop: (int) A number of loop.
         :param params: (DictData) A parameter that want to pass to stage
             execution.
-        :param context: (DictData)
-        :param result: (Result)
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
-        :rtype: tuple[Status, DictData, T]
+        :rtype: tuple[Result, T]
         """
         result.trace.debug(f"[STAGE]: Execute until item: {item!r}")
-        params["item"] = item
-        to: DictData = {"item": item, "stages": {}}
-        status: Status = SUCCESS
+        context: DictData = copy.deepcopy(params)
+        context.update({"loop": loop, "item": item, "stages": {}})
         next_item: T = None
         for stage in self.stages:
 
             if self.extras:
                 stage.extras = self.extras
 
+            if stage.is_skipped(params=context):
+                result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
+                stage.set_outputs(output={"skipped": True}, to=context)
+                continue
+
+            if event and event.is_set():
+                error_msg: str = (
+                    "Item-Stage was canceled from event that had set before "
+                    "stage item execution."
+                )
+                return (
+                    result.catch(
+                        status=FAILED,
+                        until={
+                            loop: {
+                                "loop": loop,
+                                "item": item,
+                                "stages": filter_func(
+                                    context.pop("stages", {})
+                                ),
+                                "errors": StageException(error_msg).to_dict(),
+                            }
+                        },
+                    ),
+                    next_item,
+                )
+
             try:
-                stage.set_outputs(
-                    stage.handler_execute(
-                        params=params,
-                        run_id=result.run_id,
-                        parent_run_id=result.parent_run_id,
-                    ).context,
-                    to=to,
+                rs: Result = stage.handler_execute(
+                    params=context,
+                    run_id=result.run_id,
+                    parent_run_id=result.parent_run_id,
+                    raise_error=True,
                 )
+                stage.set_outputs(rs.context, to=context)
                 if "item" in (
-                    outputs := stage.get_outputs(to).get("outputs", {})
+                    outputs := stage.get_outputs(context).get("outputs", {})
                 ):
                     next_item = outputs["item"]
-            except StageException as e:
-                status = FAILED
-                result.trace.error(
-                    f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
-                )
-                to.update({"errors": e.to_dict()})
+            except (StageException, UtilException) as e:
+                result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
+                raise StageException(
+                    f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
+                ) from None
 
-        context["until"][loop] = to
-        return status, context, next_item
+        return (
+            result.catch(
+                status=SUCCESS,
+                until={
+                    loop: {
+                        "loop": loop,
+                        "item": item,
+                        "stages": filter_func(context.pop("stages", {})),
+                    }
+                },
+            ),
+            next_item,
+        )
 
     def execute(
         self,
@@ -1369,22 +1486,31 @@ class UntilStage(BaseStage):  # pragma: no cov
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
-
+        result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
         item: Union[str, int, bool] = param2template(
             self.item, params, extras=self.extras
         )
-        result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
+        loop: int = 1
         track: bool = True
         exceed_loop: bool = False
-        loop: int = 1
-        context: DictData = {"until": {}}
-        statuses: list[Union[SUCCESS, FAILED]] = []
+        result.catch(status=WAIT, context={"until": {}})
         while track and not (exceed_loop := loop >= self.max_until_loop):
-            status, context, item = self.execute_item(
+
+            if event and event.is_set():
+                return result.catch(
+                    status=FAILED,
+                    context={
+                        "errors": StageException(
+                            "Stage was canceled from event that had set "
+                            "before stage until execution."
+                        ).to_dict()
+                    },
+                )
+
+            result, item = self.execute_item(
                 item=item,
                 loop=loop,
-                params=params.copy(),
-                context=context,
+                params=params,
                 result=result,
             )
 
@@ -1398,31 +1524,32 @@ class UntilStage(BaseStage):  # pragma: no cov
 
             next_track: bool = eval(
                 param2template(
-                    self.until, params | {"item": item}, extras=self.extras
+                    self.until,
+                    params | {"item": item, "loop": loop},
+                    extras=self.extras,
                 ),
                 globals() | params | {"item": item},
                 {},
             )
             if not isinstance(next_track, bool):
-                raise TypeError(
+                raise StageException(
                     "Return type of until condition does not be boolean, it"
                     f"return: {next_track!r}"
                 )
             track = not next_track
-            statuses.append(status)
 
         if exceed_loop:
             raise StageException(
                 f"The until loop was exceed {self.max_until_loop} loops"
             )
-        return result.catch(status=max(statuses), context=context)
+        return result.catch(status=SUCCESS)
 
 
 class Match(BaseModel):
     """Match model for the Case Stage."""
 
-    case: Union[str, int]
-    stage: Stage
+    case: Union[str, int] = Field(description="A match case.")
+    stage: Stage = Field(description="A stage to execution for this case.")
 
 
 class CaseStage(BaseStage):  # pragma: no cov
@@ -1533,8 +1660,8 @@ class CaseStage(BaseStage):  # pragma: no cov
 
 
 class RaiseStage(BaseStage):  # pragma: no cov
-    """Raise error stage that raise StageException that use a message field for
-    making error message before raise.
+    """Raise error stage execution that raise StageException that use a message
+    field for making error message before raise.
 
     Data Validate:
         >>> stage = {
@@ -1556,13 +1683,21 @@ class RaiseStage(BaseStage):  # pragma: no cov
         result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
-        """Raise the stage."""
+        """Raise the StageException object with the message field execution.
+
+        :param params: A parameter that want to pass before run any statement.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+        """
         if result is None:  # pragma: no cov
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
-        result.trace.info(f"[STAGE]: Raise-Execute: {self.message!r}.")
-        raise StageException(self.message)
+        message: str = param2template(self.message, params, extras=self.extras)
+        result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
+        raise StageException(message)
 
 
 # TODO: Not implement this stages yet
@@ -1570,7 +1705,7 @@ class HookStage(BaseStage):  # pragma: no cov
     """Hook stage execution."""
 
     hook: str
-    args: DictData
+    args: DictData = Field(default_factory=dict)
     callback: str
 
     def execute(
@@ -1603,9 +1738,69 @@ class DockerStage(BaseStage):  # pragma: no cov
     image: str = Field(
         description="A Docker image url with tag that want to run.",
     )
+    tag: str = Field(default="latest", description="An Docker image tag.")
     env: DictData = Field(default_factory=dict)
     volume: DictData = Field(default_factory=dict)
-    auth: DictData = Field(default_factory=dict)
+    auth: DictData = Field(
+        default_factory=dict,
+        description=(
+            "An authentication of the Docker registry that use in pulling step."
+        ),
+    )
+
+    def execute_task(
+        self,
+        result: Result,
+    ):
+        from docker import DockerClient
+        from docker.errors import ContainerError
+
+        client = DockerClient(
+            base_url="unix://var/run/docker.sock", version="auto"
+        )
+
+        resp = client.api.pull(
+            repository=f"{self.image}",
+            tag=self.tag,
+            # NOTE: For pulling from GCS.
+            # {
+            #     "username": "_json_key",
+            #     "password": credential-json-string,
+            # }
+            auth_config=self.auth,
+            stream=True,
+            decode=True,
+        )
+        for line in resp:
+            result.trace.info(f"... {line}")
+
+        unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S}"
+        container = client.containers.run(
+            image=f"{self.image}:{self.tag}",
+            name=unique_image_name,
+            environment=self.env,
+            volumes=(
+                {Path.cwd() / ".docker.logs": {"bind": "/logs", "mode": "rw"}}
+                | self.volume
+            ),
+            detach=True,
+        )
+
+        for line in container.logs(stream=True, timestamps=True):
+            result.trace.info(f"... {line.strip().decode()}")
+
+        # NOTE: This code copy from the docker package.
+        exit_status: int = container.wait()["StatusCode"]
+        if exit_status != 0:
+            out = container.logs(stdout=False, stderr=True)
+            container.remove()
+            raise ContainerError(
+                container,
+                exit_status,
+                None,
+                f"{self.image}:{self.tag}",
+                out,
+            )
 
     def execute(
         self,
@@ -1639,6 +1834,11 @@ class VirtualPyStage(PyStage):  # pragma: no cov
     ) -> Result:
         """Execute the Python statement via Python virtual environment.
 
+        Steps:
+            - Create python file.
+            - Create `.venv` and install necessary Python deps.
+            - Execution python file with uv and specific `.venv`.
+
         :param params: A parameter that want to pass before run any statement.
         :param result: (Result) A result object for keeping context and status
             data.