ddeutil-workflow 0.0.47__py3-none-any.whl → 0.0.49__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -38,13 +38,12 @@ from concurrent.futures import (
     ThreadPoolExecutor,
     as_completed,
 )
-from dataclasses import is_dataclass
 from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Any, Optional, Union, get_type_hints
+from typing import Annotated, Any, Optional, TypeVar, Union, get_type_hints
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
@@ -60,17 +59,7 @@ from .utils import (
     make_exec,
 )
 
-__all__: TupleStr = (
-    "EmptyStage",
-    "BashStage",
-    "PyStage",
-    "CallStage",
-    "TriggerStage",
-    "ForEachStage",
-    "ParallelStage",
-    "RaiseStage",
-    "Stage",
-)
+T = TypeVar("T")
 
 
 class BaseStage(BaseModel, ABC):
@@ -206,6 +195,7 @@ class BaseStage(BaseModel, ABC):
             run_id=run_id,
             parent_run_id=parent_run_id,
             id_logic=self.iden,
+            extras=self.extras,
         )
 
         try:
@@ -284,6 +274,25 @@ class BaseStage(BaseModel, ABC):
         to["stages"][_id] = {"outputs": output, **skipping, **errors}
         return to
 
+    def get_outputs(self, outputs: DictData) -> DictData:
+        """Get the outputs from stages data.
+
+        :rtype: DictData
+        """
+        if self.id is None and not dynamic(
+            "stage_default_id", extras=self.extras
+        ):
+            return {}
+
+        _id: str = (
+            param2template(self.id, params=outputs, extras=self.extras)
+            if self.id
+            else gen_id(
+                param2template(self.name, params=outputs, extras=self.extras)
+            )
+        )
+        return outputs.get("stages", {}).get(_id, {})
+
     def is_skipped(self, params: DictData | None = None) -> bool:
         """Return true if condition of this stage do not correct. This process
         use build-in eval function to execute the if-condition.
@@ -389,14 +398,15 @@ class BaseAsyncStage(BaseStage):
             run_id=run_id,
             parent_run_id=parent_run_id,
             id_logic=self.iden,
+            extras=self.extras,
         )
 
         try:
             rs: Result = await self.axecute(params, result=result, event=event)
-            if to is not None:
+            if to is not None:  # pragma: no cov
                 return self.set_outputs(rs.context, to=to)
             return rs
-        except Exception as e:
+        except Exception as e:  # pragma: no cov
             await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
 
             if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
@@ -818,7 +828,7 @@ class CallStage(BaseStage):
         has_keyword: bool = False
         call_func: TagFunc = extract_call(
             param2template(self.uses, params, extras=self.extras),
-            registries=self.extras.get("regis_call"),
+            registries=self.extras.get("registry_caller"),
         )()
 
         # VALIDATE: check input task caller parameters that exists before
@@ -856,6 +866,7 @@ class CallStage(BaseStage):
         )
 
         args = self.parse_model_args(call_func, args, result)
+
         if inspect.iscoroutinefunction(call_func):
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
@@ -904,8 +915,11 @@ class CallStage(BaseStage):
                 args[arg] = args.pop(arg.removeprefix("_"))
 
             t: Any = type_hints[arg]
-            if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
-                args[arg] = result
+
+            # NOTE: Check Result argument was passed to this caller function.
+            #
+            # if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
+            #     args[arg] = result
 
             if issubclass(t, BaseModel) and arg in args:
                 args[arg] = t.model_validate(obj=args[arg])
@@ -1141,13 +1155,59 @@ class ForEachStage(BaseStage):
     )
     concurrent: int = Field(
         default=1,
-        gt=0,
+        ge=1,
+        lt=10,
         description=(
             "A concurrent value allow to run each item at the same time. It "
             "will be sequential mode if this value equal 1."
         ),
     )
 
+    def execute_item(
+        self,
+        item: Union[str, int],
+        params: DictData,
+        context: DictData,
+        result: Result,
+    ) -> tuple[Status, DictData]:
+        """Execute foreach item from list of item.
+
+        :param item: (str | int) An item that want to execution.
+        :param params: (DictData) A parameter that want to pass to stage
+            execution.
+        :param context: (DictData)
+        :param result: (Result)
+
+        :rtype: tuple[Status, DictData]
+        """
+        result.trace.debug(f"[STAGE]: Execute foreach item: {item!r}")
+        params["item"] = item
+        to: DictData = {"item": item, "stages": {}}
+        status: Status = SUCCESS
+        for stage in self.stages:
+
+            if self.extras:
+                stage.extras = self.extras
+
+            try:
+                stage.set_outputs(
+                    stage.handler_execute(
+                        params=params,
+                        run_id=result.run_id,
+                        parent_run_id=result.parent_run_id,
+                    ).context,
+                    to=to,
+                )
+            except StageException as e:  # pragma: no cov
+                status = FAILED
+                result.trace.error(
+                    f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
+                )
+                to.update({"errors": e.to_dict()})
+
+        context["foreach"][item] = to
+        return status, context
+
     def execute(
         self,
         params: DictData,
@@ -1181,41 +1241,26 @@ class ForEachStage(BaseStage):
             )
 
         result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
-        rs: DictData = {"items": foreach, "foreach": {}}
-        status: Status = SUCCESS
-        # TODO: Implement concurrent more than 1.
-        for item in foreach:
-            result.trace.debug(f"[STAGE]: Execute foreach item: {item!r}")
-            params["item"] = item
-            context = {"stages": {}}
-
-            for stage in self.stages:
-
-                if self.extras:
-                    stage.extras = self.extras
-
-                try:
-                    stage.set_outputs(
-                        stage.handler_execute(
-                            params=params,
-                            run_id=result.run_id,
-                            parent_run_id=result.parent_run_id,
-                        ).context,
-                        to=context,
-                    )
-                except StageException as e:  # pragma: no cov
-                    status = FAILED
-                    result.trace.error(
-                        f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
-                    )
-                    context.update({"errors": e.to_dict()})
-
-            rs["foreach"][item] = context
+        context: DictData = {"items": foreach, "foreach": {}}
+        statuses: list[Union[SUCCESS, FAILED]] = []
+        with ThreadPoolExecutor(max_workers=self.concurrent) as executor:
+            futures: list[Future] = [
+                executor.submit(
+                    self.execute_item,
+                    item=item,
+                    params=params.copy(),
+                    context=context,
+                    result=result,
+                )
+                for item in foreach
+            ]
+            for future in as_completed(futures):
+                status, context = future.result()
+                statuses.append(status)
 
-        return result.catch(status=status, context=rs)
+        return result.catch(status=max(statuses), context=context)
 
 
-# TODO: Not implement this stages yet
 class UntilStage(BaseStage):  # pragma: no cov
     """Until execution stage.
 
@@ -1242,27 +1287,144 @@ class UntilStage(BaseStage):  # pragma: no cov
             "correct."
         ),
     )
-    max_until_not_change: int = Field(
-        default=3,
-        description="The maximum value of loop if condition not change.",
+    max_until_loop: int = Field(
+        default=10,
+        ge=1,
+        lt=100,
+        description="The maximum value of loop for this until stage.",
     )
 
+    def execute_item(
+        self,
+        item: T,
+        loop: int,
+        params: DictData,
+        context: DictData,
+        result: Result,
+    ) -> tuple[Status, DictData, T]:
+        """Execute until item set item by some stage or by default loop
+        variable.
+
+        :param item: (T) An item that want to execution.
+        :param loop: (int) A number of loop.
+        :param params: (DictData) A parameter that want to pass to stage
+            execution.
+        :param context: (DictData)
+        :param result: (Result)
+
+        :rtype: tuple[Status, DictData, T]
+        """
+        result.trace.debug(f"[STAGE]: Execute until item: {item!r}")
+        params["item"] = item
+        to: DictData = {"item": item, "stages": {}}
+        status: Status = SUCCESS
+        next_item: T = None
+        for stage in self.stages:
+
+            if self.extras:
+                stage.extras = self.extras
+
+            try:
+                stage.set_outputs(
+                    stage.handler_execute(
+                        params=params,
+                        run_id=result.run_id,
+                        parent_run_id=result.parent_run_id,
+                    ).context,
+                    to=to,
+                )
+                if "item" in (
+                    outputs := stage.get_outputs(to).get("outputs", {})
+                ):
+                    next_item = outputs["item"]
+            except StageException as e:
+                status = FAILED
+                result.trace.error(
+                    f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
+                )
+                to.update({"errors": e.to_dict()})
+
+        context["until"][loop] = to
+        return status, context, next_item
+
     def execute(
         self,
         params: DictData,
         *,
         result: Result | None = None,
         event: Event | None = None,
-    ) -> Result: ...
+    ) -> Result:
+        """Execute the stages that pass item from until condition field and
+        setter step.
+
+        :param params: A parameter that want to pass before run any statement.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
+        item: Union[str, int, bool] = param2template(
+            self.item, params, extras=self.extras
+        )
+        result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
+        track: bool = True
+        exceed_loop: bool = False
+        loop: int = 1
+        context: DictData = {"until": {}}
+        statuses: list[Union[SUCCESS, FAILED]] = []
+        while track and not (exceed_loop := loop >= self.max_until_loop):
+            status, context, item = self.execute_item(
+                item=item,
+                loop=loop,
+                params=params.copy(),
+                context=context,
+                result=result,
+            )
+
+            loop += 1
+            if item is None:
+                result.trace.warning(
+                    "... Does not have set item stage. It will use loop by "
+                    "default."
+                )
+                item = loop
+
+            next_track: bool = eval(
+                param2template(
+                    self.until, params | {"item": item}, extras=self.extras
+                ),
+                globals() | params | {"item": item},
+                {},
+            )
+            if not isinstance(next_track, bool):
+                raise TypeError(
+                    "Return type of until condition does not be boolean, it"
+                    f"return: {next_track!r}"
+                )
+            track = not next_track
+            statuses.append(status)
+
+        if exceed_loop:
+            raise StageException(
+                f"The until loop was exceed {self.max_until_loop} loops"
+            )
+        return result.catch(status=max(statuses), context=context)
 
 
-# TODO: Not implement this stages yet
 class Match(BaseModel):
+    """Match model for the Case Stage."""
+
     case: Union[str, int]
     stage: Stage
 
 
-# TODO: Not implement this stages yet
 class CaseStage(BaseStage):  # pragma: no cov
     """Case execution stage.
 
@@ -1298,7 +1460,9 @@ class CaseStage(BaseStage):  # pragma: no cov
     """
 
     case: str = Field(description="A case condition for routing.")
-    match: list[Match]
+    match: list[Match] = Field(
+        description="A list of Match model that should not be an empty list.",
+    )
 
     def execute(
         self,
@@ -1321,10 +1485,14 @@ class CaseStage(BaseStage):  # pragma: no cov
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
-        status = SUCCESS
+
         _case = param2template(self.case, params, extras=self.extras)
+
+        result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
         _else = None
+        stage: Optional[Stage] = None
         context = {}
+        status = SUCCESS
         for match in self.match:
             if (c := match.case) != "_":
                 _condition = param2template(c, params, extras=self.extras)
@@ -1332,27 +1500,35 @@ class CaseStage(BaseStage):  # pragma: no cov
                 _else = match
                 continue
 
-            if match == _condition:
+            if stage is None and _case == _condition:
                 stage: Stage = match.stage
-                if self.extras:
-                    stage.extras = self.extras
-
-                try:
-                    stage.set_outputs(
-                        stage.handler_execute(
-                            params=params,
-                            run_id=result.run_id,
-                            parent_run_id=result.parent_run_id,
-                        ).context,
-                        to=context,
-                    )
-                except StageException as e:  # pragma: no cov
-                    status = FAILED
-                    result.trace.error(
-                        f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
-                    )
-                    context.update({"errors": e.to_dict()})
 
+        if stage is None:
+            if _else is None:
+                raise StageException(
+                    "This stage does not set else for support not match "
+                    "any case."
+                )
+
+            stage: Stage = _else.stage
+
+        if self.extras:
+            stage.extras = self.extras
+
+        try:
+            context.update(
+                stage.handler_execute(
+                    params=params,
+                    run_id=result.run_id,
+                    parent_run_id=result.parent_run_id,
+                ).context
+            )
+        except StageException as e:  # pragma: no cov
+            status = FAILED
+            result.trace.error(
+                f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
+            )
+            context.update({"errors": e.to_dict()})
         return result.catch(status=status, context=context)
 
 
@@ -1403,7 +1579,8 @@ class HookStage(BaseStage):  # pragma: no cov
         *,
         result: Result | None = None,
         event: Event | None = None,
-    ) -> Result: ...
+    ) -> Result:
+        raise NotImplementedError("Hook Stage does not implement yet.")
 
 
 # TODO: Not implement this stages yet
@@ -1436,15 +1613,20 @@ class DockerStage(BaseStage):  # pragma: no cov
         *,
         result: Result | None = None,
         event: Event | None = None,
-    ) -> Result: ...
+    ) -> Result:
+        raise NotImplementedError("Docker Stage does not implement yet.")
 
 
 # TODO: Not implement this stages yet
 class VirtualPyStage(PyStage):  # pragma: no cov
     """Python Virtual Environment stage execution."""
 
-    run: str
-    vars: DictData
+    deps: list[str] = Field(
+        description=(
+            "list of Python dependency that want to install before execution "
+            "stage."
+        ),
+    )
 
     def create_py_file(self, py: str, run_id: str | None): ...
 
@@ -1455,7 +1637,25 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
-        return super().execute(params, result=result)
+        """Execute the Python statement via Python virtual environment.
+
+        :param params: A parameter that want to pass before run any statement.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
+        result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
+        raise NotImplementedError(
+            "Python Virtual Stage does not implement yet."
+        )
 
 
 # NOTE:
@@ -1465,11 +1665,16 @@ class VirtualPyStage(PyStage):  # pragma: no cov
 #
 Stage = Annotated[
     Union[
+        DockerStage,
         BashStage,
         CallStage,
+        HookStage,
         TriggerStage,
         ForEachStage,
+        UntilStage,
         ParallelStage,
+        CaseStage,
+        VirtualPyStage,
         PyStage,
         RaiseStage,
         EmptyStage,

ddeutil/workflow/utils.py

@@ -15,7 +15,7 @@ from inspect import isfunction
 from itertools import chain, islice, product
 from pathlib import Path
 from random import randrange
-from typing import Any, TypeVar
+from typing import Any, Final, TypeVar
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import hash_str
@@ -23,7 +23,7 @@ from ddeutil.core import hash_str
 from .__types import DictData, Matrix
 
 T = TypeVar("T")
-UTC = ZoneInfo("UTC")
+UTC: Final[ZoneInfo] = ZoneInfo("UTC")
 
 
 def get_dt_now(
@@ -142,7 +142,7 @@ def gen_id(
     if not isinstance(value, str):
         value: str = str(value)
 
-    if config.gen_id_simple_mode:
+    if config.generate_id_simple_mode:
         return (
             f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}T" if unique else ""
         ) + hash_str(f"{(value if sensitive else value.lower())}", n=10)