ddeutil-workflow 0.0.59__py3-none-any.whl → 0.0.61__py3-none-any.whl

ddeutil/workflow/params.py

@@ -20,6 +20,7 @@ from typing import Annotated, Any, Literal, Optional, TypeVar, Union
 from ddeutil.core import str2dict, str2list
 from pydantic import BaseModel, Field
 
+from .__types import StrOrInt
 from .exceptions import ParamValueException
 from .utils import get_d_now, get_dt_now
 
@@ -159,10 +160,11 @@ class StrParam(DefaultParam):
 
     type: Literal["str"] = "str"
 
-    def receive(self, value: Optional[str] = None) -> Optional[str]:
+    def receive(self, value: Optional[Any] = None) -> Optional[str]:
         """Receive value that match with str.
 
-        :param value: A value that want to validate with string parameter type.
+        :param value: (Any) A value that want to validate with string parameter
+            type.
         :rtype: Optional[str]
         """
         if value is None:
@@ -175,7 +177,7 @@ class IntParam(DefaultParam):
 
     type: Literal["int"] = "int"
 
-    def receive(self, value: Optional[int] = None) -> Optional[int]:
+    def receive(self, value: Optional[StrOrInt] = None) -> Optional[int]:
         """Receive value that match with int.
 
         :param value: A value that want to validate with integer parameter type.
@@ -200,13 +202,24 @@ class FloatParam(DefaultParam):  # pragma: no cov
     precision: int = 6
 
     def rounding(self, value: float) -> float:
-        """Rounding float value with the specific precision field."""
+        """Rounding float value with the specific precision field.
+
+        :param value: A float value that want to round with the precision value.
+
+        :rtype: float
+        """
         round_str: str = f"{{0:.{self.precision}f}}"
         return float(round_str.format(round(value, self.precision)))
 
-    def receive(self, value: Optional[Union[float, int, str]] = None) -> float:
+    def receive(
+        self, value: Optional[Union[float, int, str]] = None
+    ) -> Optional[float]:
+        """Receive value that match with float.
 
-        if value in None:
+        :param value: A value that want to validate with float parameter type.
+        :rtype: float | None
+        """
+        if value is None:
             return self.default
 
         if isinstance(value, float):
@@ -217,11 +230,7 @@ class FloatParam(DefaultParam):  # pragma: no cov
             raise TypeError(
                 "Received value type does not math with str, float, or int."
             )
-
-        try:
-            return self.rounding(float(value))
-        except Exception:
-            raise
+        return self.rounding(float(value))
 
 
 class DecimalParam(DefaultParam):  # pragma: no cov
@@ -231,12 +240,28 @@ class DecimalParam(DefaultParam):  # pragma: no cov
     precision: int = 6
 
     def rounding(self, value: Decimal) -> Decimal:
-        """Rounding float value with the specific precision field."""
+        """Rounding float value with the specific precision field.
+
+        :param value: (Decimal) A Decimal value that want to round with the
+            precision value.
+
+        :rtype: Decimal
+        """
         return value.quantize(Decimal(10) ** -self.precision)
 
-    def receive(self, value: float | Decimal | None = None) -> Decimal:
+    def receive(
+        self, value: Optional[Union[float, int, str, Decimal]] = None
+    ) -> Decimal:
+        """Receive value that match with decimal.
 
-        if isinstance(value, float):
+        :param value: (float | Decimal) A value that want to validate with
+            decimal parameter type.
+        :rtype: Decimal | None
+        """
+        if value is None:
+            return self.default
+
+        if isinstance(value, (float, int)):
             return self.rounding(Decimal(value))
         elif isinstance(value, Decimal):
             return self.rounding(value)
@@ -261,11 +286,12 @@ class ChoiceParam(BaseParam):
         description="A list of choice parameters that able be str or int.",
     )
 
-    def receive(self, value: Union[str, int] | None = None) -> Union[str, int]:
+    def receive(self, value: Optional[StrOrInt] = None) -> StrOrInt:
         """Receive value that match with options.
 
-        :param value: A value that want to select from the options field.
-        :rtype: str
+        :param value: (str | int) A value that want to select from the options
+            field.
+        :rtype: str | int
         """
         # NOTE:
         #   Return the first value in options if it does not pass any input
@@ -279,7 +305,7 @@ class ChoiceParam(BaseParam):
         return value
 
 
-class MapParam(DefaultParam):  # pragma: no cov
+class MapParam(DefaultParam):
     """Map parameter."""
 
     type: Literal["map"] = "map"
@@ -295,6 +321,7 @@ class MapParam(DefaultParam):  # pragma: no cov
         """Receive value that match with map type.
 
         :param value: A value that want to validate with map parameter type.
+
         :rtype: dict[Any, Any]
         """
         if value is None:
@@ -316,7 +343,7 @@ class MapParam(DefaultParam):  # pragma: no cov
         return value
 
 
-class ArrayParam(DefaultParam):  # pragma: no cov
+class ArrayParam(DefaultParam):
     """Array parameter."""
 
     type: Literal["array"] = "array"
@@ -326,7 +353,7 @@ class ArrayParam(DefaultParam):  # pragma: no cov
     )
 
     def receive(
-        self, value: Optional[Union[list[T], tuple[T, ...], str]] = None
+        self, value: Optional[Union[list[T], tuple[T, ...], set[T], str]] = None
     ) -> list[T]:
         """Receive value that match with array type.
 
@@ -365,5 +392,11 @@ Param = Annotated[
         IntParam,
         StrParam,
     ],
-    Field(discriminator="type"),
+    Field(
+        discriminator="type",
+        description=(
+            "A parameter models that use for validate and receive on the "
+            "workflow execution."
+        ),
+    ),
 ]

ddeutil/workflow/result.py

@@ -38,7 +38,7 @@ class Status(IntEnum):
     CANCEL = 4
 
     @property
-    def emoji(self) -> str:
+    def emoji(self) -> str:  # pragma: no cov
         """Return the emoji value of this status.
 
         :rtype: str

ddeutil/workflow/reusables.py

@@ -44,6 +44,8 @@ FILTERS: dict[str, Callable] = {  # pragma: no cov
     "upper": lambda x: x.upper(),
     "lower": lambda x: x.lower(),
     "rstr": [str, repr],
+    "keys": lambda x: list(x.keys()),
+    "values": lambda x: list(x.values()),
 }
 
 

ddeutil/workflow/stages.py

@@ -35,6 +35,7 @@ import json
 import subprocess
 import sys
 import time
+import traceback
 import uuid
 from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator, Iterator
@@ -58,19 +59,21 @@ from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, StrOrInt, TupleStr
+from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
 from .conf import dynamic
 from .exceptions import StageException, to_dict
 from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
 from .reusables import TagFunc, extract_call, not_in_template, param2template
 from .utils import (
     delay,
+    dump_all,
     filter_func,
     gen_id,
     make_exec,
 )
 
 T = TypeVar("T")
+DictOrModel = Union[DictData, BaseModel]
 
 
 class BaseStage(BaseModel, ABC):
@@ -87,7 +90,7 @@ class BaseStage(BaseModel, ABC):
         default_factory=dict,
         description="An extra parameter that override core config values.",
     )
-    id: Optional[str] = Field(
+    id: StrOrNone = Field(
         default=None,
         description=(
             "A stage ID that use to keep execution output or getting by job "
@@ -97,7 +100,7 @@ class BaseStage(BaseModel, ABC):
     name: str = Field(
         description="A stage name that want to logging when start execution.",
     )
-    condition: Optional[str] = Field(
+    condition: StrOrNone = Field(
         default=None,
         description=(
             "A stage condition statement to allow stage executable. This field "
@@ -162,8 +165,8 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        run_id: Optional[str] = None,
-        parent_run_id: Optional[str] = None,
+        run_id: StrOrNone = None,
+        parent_run_id: StrOrNone = None,
         result: Optional[Result] = None,
         event: Optional[Event] = None,
         raise_error: Optional[bool] = None,
@@ -221,7 +224,10 @@ class BaseStage(BaseModel, ABC):
             return self.execute(params, result=result, event=event)
         except Exception as e:
             e_name: str = e.__class__.__name__
-            result.trace.error(f"[STAGE]: Error Handler:||{e_name}:||{e}")
+            result.trace.error(
+                f"[STAGE]: Error Handler:||{e_name}:||{e}||"
+                f"{traceback.format_exc()}"
+            )
             if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
                 if isinstance(e, StageException):
                     raise
@@ -411,8 +417,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        run_id: Optional[str] = None,
-        parent_run_id: Optional[str] = None,
+        run_id: StrOrNone = None,
+        parent_run_id: StrOrNone = None,
         result: Optional[Result] = None,
         event: Optional[Event] = None,
         raise_error: Optional[bool] = None,
@@ -469,7 +475,7 @@ class EmptyStage(BaseAsyncStage):
         ... }
     """
 
-    echo: Optional[str] = Field(
+    echo: StrOrNone = Field(
         default=None,
         description="A message that want to show on the stdout.",
     )
@@ -598,14 +604,14 @@ class BashStage(BaseAsyncStage):
     )
 
     @contextlib.asynccontextmanager
-    async def acreate_sh_file(
-        self, bash: str, env: DictStr, run_id: Optional[str] = None
+    async def async_create_sh_file(
+        self, bash: str, env: DictStr, run_id: StrOrNone = None
     ) -> AsyncIterator[TupleStr]:
         """Async create and write `.sh` file with the `aiofiles` package.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (Optional[str]) A running stage ID that use for writing sh
+        :param run_id: (StrOrNone) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: AsyncIterator[TupleStr]
@@ -635,14 +641,14 @@ class BashStage(BaseAsyncStage):
 
     @contextlib.contextmanager
     def create_sh_file(
-        self, bash: str, env: DictStr, run_id: Optional[str] = None
+        self, bash: str, env: DictStr, run_id: StrOrNone = None
     ) -> Iterator[TupleStr]:
         """Create and write the `.sh` file before giving this file name to
         context. After that, it will auto delete this file automatic.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (Optional[str]) A running stage ID that use for writing sh
+        :param run_id: (StrOrNone) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: Iterator[TupleStr]
@@ -752,7 +758,7 @@ class BashStage(BaseAsyncStage):
             dedent(self.bash.strip("\n")), params, extras=self.extras
         )
 
-        async with self.acreate_sh_file(
+        async with self.async_create_sh_file(
             bash=bash,
             env=param2template(self.env, params, extras=self.extras),
             run_id=result.run_id,
@@ -1170,13 +1176,12 @@ class CallStage(BaseAsyncStage):
             args.pop("result")
 
         args = self.parse_model_args(call_func, args, result)
-
         if inspect.iscoroutinefunction(call_func):
-            rs: DictData = await call_func(
+            rs: DictOrModel = await call_func(
                 **param2template(args, params, extras=self.extras)
             )
         else:
-            rs: DictData = call_func(
+            rs: DictOrModel = call_func(
                 **param2template(args, params, extras=self.extras)
             )
 
@@ -1190,7 +1195,7 @@ class CallStage(BaseAsyncStage):
                 f"serialize, you must set return be `dict` or Pydantic "
                 f"model."
             )
-        return result.catch(status=SUCCESS, context=rs)
+        return result.catch(status=SUCCESS, context=dump_all(rs, by_alias=True))
 
     @staticmethod
     def parse_model_args(
@@ -1294,11 +1299,12 @@ class TriggerStage(BaseStage):
             extras=self.extras | {"stage_raise_error": True},
         ).execute(
             params=param2template(self.params, params, extras=self.extras),
-            parent_run_id=result.run_id,
+            run_id=None,
+            parent_run_id=result.parent_run_id,
             event=event,
         )
         if rs.status == FAILED:
-            err_msg: Optional[str] = (
+            err_msg: StrOrNone = (
                 f" with:\n{msg}"
                 if (msg := rs.context.get("errors", {}).get("message"))
                 else "."
@@ -1826,7 +1832,10 @@ class UntilStage(BaseNestedStage):
         ...     "stages": [
         ...         {
         ...             "name": "Start increase item value.",
-        ...             "run": "item = ${{ item }}\\nitem += 1\\n"
+        ...             "run": (
+        ...                 "item = ${{ item }}\\n"
+        ...                 "item += 1\\n"
+        ...             )
         ...         },
         ...     ],
         ... }
@@ -2215,9 +2224,7 @@ class CaseStage(BaseNestedStage):
             extras=self.extras,
         )
 
-        _case: Optional[str] = param2template(
-            self.case, params, extras=self.extras
-        )
+        _case: StrOrNone = param2template(self.case, params, extras=self.extras)
 
         result.trace.info(f"[STAGE]: Execute Case-Stage: {_case!r}.")
         _else: Optional[Match] = None
@@ -2396,8 +2403,14 @@ class DockerStage(BaseStage):  # pragma: no cov
 
         :rtype: Result
         """
-        from docker import DockerClient
-        from docker.errors import ContainerError
+        try:
+            from docker import DockerClient
+            from docker.errors import ContainerError
+        except ImportError:
+            raise ImportError(
+                "Docker stage need the docker package, you should install it "
+                "by `pip install docker` first."
+            ) from None
 
         client = DockerClient(
             base_url="unix://var/run/docker.sock", version="auto"
@@ -2459,7 +2472,7 @@ class DockerStage(BaseStage):  # pragma: no cov
                 exit_status,
                 None,
                 f"{self.image}:{self.tag}",
-                out,
+                out.decode("utf-8"),
             )
         output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
         if not output_file.exists():
@@ -2518,15 +2531,19 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         py: str,
         values: DictData,
         deps: list[str],
-        run_id: Optional[str] = None,
+        run_id: StrOrNone = None,
     ) -> Iterator[str]:
-        """Create the .py file with an input Python string statement.
+        """Create the `.py` file and write an input Python statement and its
+        Python dependency on the header of this file.
+
+            The format of Python dependency was followed by the `uv`
+        recommended.
 
         :param py: A Python string statement.
         :param values: A variable that want to set before running this
         :param deps: An additional Python dependencies that want install before
             run this python stage.
-        :param run_id: (Optional[str]) A running ID of this stage execution.
+        :param run_id: (StrOrNone) A running ID of this stage execution.
         """
         run_id: str = run_id or uuid.uuid4()
         f_name: str = f"{run_id}.py"
@@ -2536,7 +2553,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
                 f"{var} = {value!r}" for var, value in values.items()
             )
 
-            # NOTE: uv supports PEP 723 — inline TOML metadata.
+            # NOTE: `uv` supports PEP 723 — inline TOML metadata.
             f.write(
                 dedent(
                     f"""
@@ -2595,6 +2612,16 @@ class VirtualPyStage(PyStage):  # pragma: no cov
             run_id=result.run_id,
         ) as py:
             result.trace.debug(f"[STAGE]: ... Create `{py}` file.")
+            try:
+                import uv
+
+                _ = uv
+            except ImportError:
+                raise ImportError(
+                    "The VirtualPyStage need you to install `uv` before"
+                    "execution."
+                ) from None
+
             rs: CompletedProcess = subprocess.run(
                 ["uv", "run", py, "--no-cache"],
                 # ["uv", "run", "--python", "3.9", py],
@@ -2644,5 +2671,8 @@ Stage = Annotated[
         RaiseStage,
         EmptyStage,
     ],
-    Field(union_mode="smart"),
+    Field(
+        union_mode="smart",
+        description="A stage models that already implemented on this package.",
+    ),
 ]  # pragma: no cov

ddeutil/workflow/utils.py

@@ -15,16 +15,17 @@ from inspect import isfunction
 from itertools import chain, islice, product
 from pathlib import Path
 from random import randrange
-from typing import Any, Final, Optional, TypeVar, Union
+from typing import Any, Final, Optional, TypeVar, Union, overload
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import hash_str
+from pydantic import BaseModel
 
 from .__types import DictData, Matrix
 
 T = TypeVar("T")
 UTC: Final[ZoneInfo] = ZoneInfo("UTC")
-MARK_NL: Final[str] = "||"
+MARK_NEWLINE: Final[str] = "||"
 
 
 def prepare_newline(msg: str) -> str:
@@ -34,11 +35,12 @@ def prepare_newline(msg: str) -> str:
 
     :rtype: str
     """
-    msg: str = msg.strip("\n").replace("\n", MARK_NL)
-    if MARK_NL not in msg:
+    # NOTE: Remove ending with "\n" and replace "\n" with the "||" value.
+    msg: str = msg.strip("\n").replace("\n", MARK_NEWLINE)
+    if MARK_NEWLINE not in msg:
         return msg
 
-    msg_lines: list[str] = msg.split(MARK_NL)
+    msg_lines: list[str] = msg.split(MARK_NEWLINE)
     msg_last: str = msg_lines[-1]
     msg_body: str = (
         "\n" + "\n".join(f" ... |  \t{s}" for s in msg_lines[1:-1])
@@ -288,3 +290,24 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
         dt, simple = run_id.split("T", maxsplit=1)
         return dt[:12] + simple[-num:]
     return run_id[:12] + run_id[-num:]
+
+
+@overload
+def dump_all(value: BaseModel, by_alias: bool = False) -> DictData: ...
+
+
+@overload
+def dump_all(value: T, by_alias: bool = False) -> T: ...
+
+
+def dump_all(
+    value: Union[T, BaseModel], by_alias: bool = False
+) -> Union[T, DictData]:
+    """Dump all BaseModel object to dict."""
+    if isinstance(value, dict):
+        return {k: dump_all(value[k], by_alias=by_alias) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([dump_all(i, by_alias=by_alias) for i in value])
+    elif isinstance(value, BaseModel):
+        return value.model_dump(by_alias=by_alias)
+    return value