ddeutil-workflow 0.0.42__py3-none-any.whl → 0.0.44__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.42"
+__version__: str = "0.0.44"

ddeutil/workflow/__init__.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
-from .__types import Re
+from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import (
     Config,
     Loader,
@@ -47,6 +47,10 @@ from .params import (
     StrParam,
 )
 from .result import (
+    FAILED,
+    SKIP,
+    SUCCESS,
+    WAIT,
     Result,
     Status,
 )

ddeutil/workflow/exceptions.py

@@ -9,10 +9,20 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
-from typing import Any
+from typing import TypedDict
 
+ErrorData = TypedDict(
+    "ErrorData",
+    {
+        "class": Exception,
+        "name": str,
+        "message": str,
+    },
+)
 
-def to_dict(exception: Exception) -> dict[str, Any]:  # pragma: no cov
+
+def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
+    """Create dict data from exception instance."""
     return {
         "class": exception,
         "name": exception.__class__.__name__,
@@ -22,7 +32,7 @@ def to_dict(exception: Exception) -> dict[str, Any]:  # pragma: no cov
 
 class BaseWorkflowException(Exception):
 
-    def to_dict(self) -> dict[str, Any]:
+    def to_dict(self) -> ErrorData:
         return to_dict(self)
 
 

ddeutil/workflow/job.py

@@ -38,8 +38,9 @@ from .exceptions import (
     JobException,
     StageException,
     UtilException,
+    to_dict,
 )
-from .result import Result, Status
+from .result import FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
 from .stages import Stage
 from .utils import cross_product, filter_func, gen_id
@@ -51,7 +52,6 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
-    "TriggerState",
     "RunsOn",
     "RunsOnLocal",
     "RunsOnSelfHosted",
@@ -206,16 +206,6 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
-class TriggerState(str, Enum):
-    waiting: str = "waiting"
-    passed: str = "passed"
-    skipped: str = "skipped"
-    failed: str = "failed"
-
-    def is_waiting(self):
-        return self.value == "waiting"
-
-
 class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
@@ -407,30 +397,32 @@ class Job(BaseModel):
     def check_needs(
         self,
         jobs: dict[str, Any],
-    ) -> TriggerState:  # pragma: no cov
-        """Return True if job's need exists in an input list of job's ID.
+    ) -> Status:  # pragma: no cov
+        """Return Status enum for checking job's need trigger logic in an
+        input list of job's ID.
 
         :param jobs: A mapping of job ID and result context.
 
         :raise NotImplementedError: If the job trigger rule out of scope.
 
-        :rtype: TriggerState
+        :rtype: Status
         """
         if not self.needs:
-            return TriggerState.passed
+            return SUCCESS
 
-        def make_return(result: bool) -> TriggerState:
-            return TriggerState.passed if result else TriggerState.failed
+        def make_return(result: bool) -> Status:
+            return SUCCESS if result else FAILED
 
         need_exist: dict[str, Any] = {
             need: jobs[need] for need in self.needs if need in jobs
         }
+
         if len(need_exist) != len(self.needs):
-            return TriggerState.waiting
+            return WAIT
         elif all("skipped" in need_exist[job] for job in need_exist):
-            return TriggerState.skipped
+            return SKIP
         elif self.trigger_rule == TriggerRules.all_done:
-            return TriggerState.passed
+            return SUCCESS
         elif self.trigger_rule == TriggerRules.all_success:
             rs = all(
                 k not in need_exist[job]
@@ -640,19 +632,6 @@ def local_execute_strategy(
         result: Result = Result(run_id=gen_id(job.id or "not-set", unique=True))
 
     strategy_id: str = gen_id(strategy)
-
-    # PARAGRAPH:
-    #
-    #       Create strategy execution context and update a matrix and copied
-    #   of params. So, the context value will have structure like;
-    #
-    #   {
-    #       "params": { ... },      <== Current input params
-    #       "jobs": { ... },        <== Current input params
-    #       "matrix": { ... }       <== Current strategy value
-    #       "stages": { ... }       <== Catching stage outputs
-    #   }
-    #
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
 
@@ -660,7 +639,6 @@ def local_execute_strategy(
         result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
         result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
 
-    # IMPORTANT: The stage execution only run sequentially one-by-one.
     for stage in job.stages:
 
         if stage.is_skipped(params=context):
@@ -674,7 +652,7 @@ def local_execute_strategy(
                 "strategy execution."
             )
             return result.catch(
-                status=Status.FAILED,
+                status=FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
@@ -684,34 +662,30 @@ def local_execute_strategy(
                 },
             )
 
-        # PARAGRAPH:
-        #
-        #       This step will add the stage result to `stages` key in that
-        #   stage id. It will have structure like;
-        #
-        #   {
-        #       "params": { ... },
-        #       "jobs": { ... },
-        #       "matrix": { ... },
-        #       "stages": { { "stage-id-01": { "outputs": { ... } } }, ... }
-        #   }
-        #
-        # IMPORTANT:
-        #   This execution change all stage running IDs to the current job
-        #   running ID, but it still trac log to the same parent running ID
-        #   (with passing `run_id` and `parent_run_id` to the stage
-        #   execution arguments).
-        #
         try:
-            stage.set_outputs(
-                stage.handler_execute(
-                    params=context,
-                    run_id=result.run_id,
-                    parent_run_id=result.parent_run_id,
-                    event=event,
-                ).context,
-                to=context,
+            rs: Result = stage.handler_execute(
+                params=context,
+                run_id=result.run_id,
+                parent_run_id=result.parent_run_id,
+                event=event,
             )
+            stage.set_outputs(rs.context, to=context)
+            if rs.status == FAILED:
+                error_msg: str = (
+                    f"Job strategy was break because it has a stage, "
+                    f"{stage.iden}, failed without raise error."
+                )
+                return result.catch(
+                    status=FAILED,
+                    context={
+                        strategy_id: {
+                            "matrix": strategy,
+                            "stages": context.pop("stages", {}),
+                            "errors": JobException(error_msg).to_dict(),
+                        },
+                    },
+                )
+
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
             do_raise: bool = dynamic(
@@ -724,7 +698,7 @@ def local_execute_strategy(
                 ) from None
 
             return result.catch(
-                status=Status.FAILED,
+                status=FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
@@ -735,7 +709,7 @@ def local_execute_strategy(
             )
 
     return result.catch(
-        status=Status.SUCCESS,
+        status=SUCCESS,
         context={
             strategy_id: {
                 "matrix": strategy,
@@ -756,8 +730,8 @@ def local_execute(
     raise_error: bool | None = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
-    execution. It will generate matrix values at the first step and run
-    multithread on this metrics to the `stages` field of this job.
+    execution or itself execution. It will generate matrix values at the first
+    step and run multithread on this metrics to the `stages` field of this job.
 
         This method does not raise any JobException if it runs with
     multi-threading strategy.
@@ -790,7 +764,7 @@ def local_execute(
 
             if event and event.is_set():  # pragma: no cov
                 return result.catch(
-                    status=Status.FAILED,
+                    status=FAILED,
                     context={
                         "errors": JobException(
                             "Job strategy was canceled from event that had set "
@@ -808,7 +782,7 @@ def local_execute(
                 raise_error=raise_error,
             )
 
-        return result.catch(status=Status.SUCCESS)
+        return result.catch(status=result.status)
 
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
@@ -819,7 +793,7 @@ def local_execute(
 
     if event and event.is_set():  # pragma: no cov
         return result.catch(
-            status=Status.FAILED,
+            status=FAILED,
             context={
                 "errors": JobException(
                     "Job strategy was canceled from event that had set "
@@ -828,8 +802,6 @@ def local_execute(
             },
         )
 
-    # IMPORTANT: Start running strategy execution by multithreading because
-    #   it will run by strategy values without waiting previous execution.
     with ThreadPoolExecutor(
         max_workers=job.strategy.max_parallel,
         thread_name_prefix="job_strategy_exec_",
@@ -849,7 +821,7 @@ def local_execute(
         ]
 
         context: DictData = {}
-        status: Status = Status.SUCCESS
+        status: Status = SUCCESS
 
         if not fail_fast_flag:
             done = as_completed(futures, timeout=1800)
@@ -875,7 +847,7 @@ def local_execute(
             try:
                 future.result()
             except JobException as err:
-                status = Status.FAILED
+                status = FAILED
                 result.trace.error(
                     f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
                     f"\n\t{err}"
@@ -895,6 +867,22 @@ def self_hosted_execute(
     event: Event | None = None,
     raise_error: bool | None = None,
 ) -> Result:  # pragma: no cov
+    """Self-Hosted job execution with passing dynamic parameters from the
+    workflow execution or itself execution. It will make request to the
+    self-hosted host url.
+
+    :param job: (Job) A job model that want to execute.
+    :param params: (DictData) An input parameters that use on job execution.
+    :param run_id: (str) A job running ID for this execution.
+    :param parent_run_id: (str) A parent workflow running ID for this release.
+    :param result: (Result) A result object for keeping context and status
+        data.
+    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error to the
+        strategy execution.
+
+    :rtype: Result
+    """
     result: Result = Result.construct_with_rs_or_id(
         result,
         run_id=run_id,
@@ -903,14 +891,31 @@ def self_hosted_execute(
     )
 
     if event and event.is_set():
-        return result.catch(status=Status.FAILED)
+        return result.catch(
+            status=FAILED,
+            context={
+                "errors": JobException(
+                    "Job self-hosted execution was canceled from event that "
+                    "had set before start execution."
+                ).to_dict()
+            },
+        )
 
     import requests
 
-    resp = requests.post(
-        job.runs_on.args.host,
-        data={"job": job.model_dump(), "params": params},
-    )
+    try:
+        resp = requests.post(
+            job.runs_on.args.host,
+            headers={"Auth": f"Barer {job.runs_on.args.token}"},
+            data={
+                "job": job.model_dump(),
+                "params": params,
+                "result": result.__dict__,
+                "raise_error": raise_error,
+            },
+        )
+    except requests.exceptions.RequestException as e:
+        return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
     if resp.status_code != 200:
         do_raise: bool = dynamic(
@@ -922,5 +927,5 @@ def self_hosted_execute(
                 f"{job.runs_on.args.host!r}"
             )
 
-        return result.catch(status=Status.FAILED)
-    return result.catch(status=Status.SUCCESS)
+        return result.catch(status=FAILED)
+    return result.catch(status=SUCCESS)

ddeutil/workflow/params.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [ ] Use config
 """This module include all Param Pydantic Models that use for parsing an
 incoming parameters that was passed to the Workflow and Schedule objects before
 execution or release methods.
@@ -18,6 +17,7 @@ from abc import ABC, abstractmethod
 from datetime import date, datetime
 from typing import Annotated, Any, Literal, Optional, TypeVar, Union
 
+from ddeutil.core import str2dict, str2list
 from pydantic import BaseModel, Field
 
 from .__types import TupleStr
@@ -31,6 +31,8 @@ __all__: TupleStr = (
     "IntParam",
     "Param",
     "StrParam",
+    "ArrayParam",
+    "MapParam",
 )
 
 T = TypeVar("T")
@@ -42,7 +44,10 @@ class BaseParam(BaseModel, ABC):
     """
 
     desc: Optional[str] = Field(
-        default=None, description="A description of parameter providing."
+        default=None,
+        description=(
+            "A description of this parameter provide to the workflow model."
+        ),
     )
     required: bool = Field(
         default=True,
@@ -52,6 +57,7 @@ class BaseParam(BaseModel, ABC):
 
     @abstractmethod
     def receive(self, value: Optional[T] = None) -> T:
+        """Abstract method receive value to this parameter model."""
         raise NotImplementedError(
             "Receive value and validate typing before return valid value."
         )
@@ -66,13 +72,14 @@ class DefaultParam(BaseParam):
         default=False,
         description="A require flag for the default-able parameter value.",
     )
-    default: Optional[str] = Field(
+    default: Optional[Any] = Field(
         default=None,
         description="A default value if parameter does not pass.",
     )
 
     @abstractmethod
     def receive(self, value: Optional[Any] = None) -> Any:
+        """Abstract method receive value to this parameter model."""
         raise NotImplementedError(
             "Receive value and validate typing before return valid value."
         )
@@ -82,7 +89,10 @@ class DateParam(DefaultParam):  # pragma: no cov
     """Date parameter model."""
 
     type: Literal["date"] = "date"
-    default: date = Field(default_factory=get_d_now)
+    default: date = Field(
+        default_factory=get_d_now,
+        description="A default date that make from the current date func.",
+    )
 
     def receive(self, value: Optional[str | datetime | date] = None) -> date:
         """Receive value that match with date. If an input value pass with
@@ -116,7 +126,12 @@ class DatetimeParam(DefaultParam):
     """Datetime parameter model."""
 
     type: Literal["datetime"] = "datetime"
-    default: datetime = Field(default_factory=get_dt_now)
+    default: datetime = Field(
+        default_factory=get_dt_now,
+        description=(
+            "A default datetime that make from the current datetime func."
+        ),
+    )
 
     def receive(self, value: str | datetime | date | None = None) -> datetime:
         """Receive value that match with datetime. If an input value pass with
@@ -167,10 +182,6 @@ class IntParam(DefaultParam):
     """Integer parameter."""
 
     type: Literal["int"] = "int"
-    default: Optional[int] = Field(
-        default=None,
-        description="A default value if parameter does not pass.",
-    )
 
     def receive(self, value: int | None = None) -> int | None:
         """Receive value that match with int.
@@ -222,36 +233,84 @@ class ChoiceParam(BaseParam):
         return value
 
 
-# TODO: Not implement this parameter yet
 class MapParam(DefaultParam):  # pragma: no cov
+    """Map parameter."""
 
     type: Literal["map"] = "map"
-    default: dict[Any, Any] = Field(default_factory=dict)
+    default: dict[Any, Any] = Field(
+        default_factory=dict,
+        description="A default dict that make from the dict built-in func.",
+    )
+
+    def receive(
+        self,
+        value: Optional[Union[dict[Any, Any], str]] = None,
+    ) -> dict[Any, Any]:
+        """Receive value that match with map type.
 
-    def receive(self, value: Optional[dict[Any, Any]] = None) -> dict[Any, Any]:
+        :param value: A value that want to validate with map parameter type.
+        :rtype: dict[Any, Any]
+        """
         if value is None:
             return self.default
 
+        if isinstance(value, str):
+            try:
+                value: dict[Any, Any] = str2dict(value)
+            except ValueError as e:
+                raise ParamValueException(
+                    f"Value that want to convert to map does not support for "
+                    f"type: {type(value)}"
+                ) from e
+        elif not isinstance(value, dict):
+            raise ParamValueException(
+                f"Value of map param support only string-dict or dict type, "
+                f"not {type(value)}"
+            )
+        return value
+
 
-# TODO: Not implement this parameter yet
 class ArrayParam(DefaultParam):  # pragma: no cov
+    """Array parameter."""
 
     type: Literal["array"] = "array"
-    default: list[Any] = Field(default_factory=list)
+    default: list[Any] = Field(
+        default_factory=list,
+        description="A default list that make from the list built-in func.",
+    )
 
-    def receive(self, value: Optional[list[T]] = None) -> list[T]:
+    def receive(
+        self, value: Optional[Union[list[T], tuple[T, ...], str]] = None
+    ) -> list[T]:
+        """Receive value that match with array type.
+
+        :param value: A value that want to validate with array parameter type.
+        :rtype: list[Any]
+        """
         if value is None:
             return self.default
-        if not isinstance(value, list):
+        if isinstance(value, str):
+            try:
+                value: list[T] = str2list(value)
+            except ValueError as e:
+                raise ParamValueException(
+                    f"Value that want to convert to array does not support for "
+                    f"type: {type(value)}"
+                ) from e
+        elif isinstance(value, (tuple, set)):
+            return list(value)
+        elif not isinstance(value, list):
             raise ParamValueException(
-                f"Value that want to convert to array does not support for "
-                f"type: {type(value)}"
+                f"Value of map param support only string-list or list type, "
+                f"not {type(value)}"
             )
         return value
 
 
 Param = Annotated[
     Union[
+        MapParam,
+        ArrayParam,
         ChoiceParam,
         DatetimeParam,
         DateParam,

ddeutil/workflow/result.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This is the Result module. It is the data context transfer objects that use
-by all object in this package. This module provide Result dataclass.
+"""Result module. It is the data context transfer objects that use by all object
+in this package. This module provide Status enum object and Result dataclass.
 """
 from __future__ import annotations
 
@@ -23,13 +23,19 @@ from .logs import TraceLog, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id
 
 __all__: TupleStr = (
+    "SUCCESS",
+    "FAILED",
+    "WAIT",
+    "SKIP",
     "Result",
     "Status",
 )
 
 
 class Status(IntEnum):
-    """Status Int Enum object."""
+    """Status Int Enum object that use for tracking execution status to the
+    Result dataclass object.
+    """
 
     SUCCESS: int = 0
     FAILED: int = 1
@@ -37,8 +43,17 @@ class Status(IntEnum):
     SKIP: int = 3
 
 
+SUCCESS = Status.SUCCESS
+FAILED = Status.FAILED
+WAIT = Status.WAIT
+SKIP = Status.SKIP
+
+
 @dataclass(
-    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+    config=ConfigDict(
+        arbitrary_types_allowed=True,
+        use_enum_values=True,
+    ),
 )
 class Result:
     """Result Pydantic Model for passing and receiving data context from any
@@ -49,8 +64,9 @@ class Result:
     and ``_run_id`` fields to comparing with other result instance.
     """
 
-    status: Status = field(default=Status.WAIT)
+    status: Status = field(default=WAIT)
     context: DictData = field(default_factory=dict)
+    errors: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
@@ -64,9 +80,16 @@ class Result:
         run_id: str | None = None,
         parent_run_id: str | None = None,
         id_logic: str | None = None,
-    ) -> Self:  # pragma: no cov
+    ) -> Self:
         """Create the Result object or set parent running id if passing Result
         object.
+
+        :param result:
+        :param run_id:
+        :param parent_run_id:
+        :param id_logic:
+
+        :rtype: Self
         """
         if result is None:
             result: Result = cls(
@@ -101,18 +124,23 @@ class Result:
         self,
         status: int | Status,
         context: DictData | None = None,
+        error: DictData | None = None,
     ) -> Self:
         """Catch the status and context to this Result object. This method will
         use between a child execution return a result, and it wants to pass
         status and context to this object.
 
-        :param status:
-        :param context:
+        :param status: A status enum object.
+        :param context: A context data that will update to the current context.
+        :param error: An error data that will update to the current errors.
+
+        :rtype: Self
         """
         self.__dict__["status"] = (
             Status(status) if isinstance(status, int) else status
         )
         self.__dict__["context"].update(context or {})
+        self.__dict__["errors"].update(error or {})
         return self
 
     def alive_time(self) -> float:  # pragma: no cov