ddeutil-workflow 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl

ddeutil/workflow/exceptions.py

@@ -9,31 +9,61 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
-from typing import TypedDict
+from typing import Literal, TypedDict, overload
 
-ErrorData = TypedDict(
-    "ErrorData",
-    {
-        "class": Exception,
-        "name": str,
-        "message": str,
-    },
-)
+
+class ErrorData(TypedDict):
+    """Error data type dict for typing necessary keys of return of to_dict func
+    and method.
+    """
+
+    name: str
+    message: str
 
 
 def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
-    """Create dict data from exception instance."""
+    """Create dict data from exception instance.
+
+    :param exception: An exception object.
+
+    :rtype: ErrorData
+    """
     return {
-        "class": exception,
         "name": exception.__class__.__name__,
         "message": str(exception),
     }
 
 
 class BaseWorkflowException(Exception):
-
-    def to_dict(self) -> ErrorData:
-        return to_dict(self)
+    """Base Workflow exception class will implement the `refs` argument for
+    making an error context to the result context.
+    """
+
+    def __init__(self, message: str, *, refs: str | None = None):
+        super().__init__(message)
+        self.refs: str | None = refs
+
+    @overload
+    def to_dict(
+        self, with_refs: Literal[True] = ...
+    ) -> dict[str, ErrorData]: ...  # pragma: no cov
+
+    @overload
+    def to_dict(
+        self, with_refs: Literal[False] = ...
+    ) -> ErrorData: ...  # pragma: no cov
+
+    def to_dict(
+        self, with_refs: bool = False
+    ) -> ErrorData | dict[str, ErrorData]:
+        """Return ErrorData data from the current exception object.
+
+        :rtype: ErrorData
+        """
+        data: ErrorData = to_dict(self)
+        if with_refs and (self.refs is not None and self.refs != "EMPTY"):
+            return {self.refs: data}
+        return data
 
 
 class UtilException(BaseWorkflowException): ...

ddeutil/workflow/job.py

@@ -18,8 +18,10 @@ method.
 from __future__ import annotations
 
 import copy
+import time
 from concurrent.futures import (
     FIRST_EXCEPTION,
+    CancelledError,
     Future,
     ThreadPoolExecutor,
     as_completed,
@@ -40,13 +42,12 @@ from .__types import DictData, DictStr, Matrix
 from .exceptions import (
     JobException,
     StageException,
-    UtilException,
     to_dict,
 )
 from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
 from .stages import Stage
-from .utils import NEWLINE, cross_product, filter_func, gen_id
+from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -140,14 +141,19 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
+        description=(
+            "A fail-fast flag that use to cancel strategy execution when it "
+            "has some execution was failed."
+        ),
         alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
         gt=0,
+        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
-            "parallel"
+            "parallel. This value should gather than 0 and less than 10."
         ),
         alias="max-parallel",
     )
@@ -375,7 +381,7 @@ class Job(BaseModel):
 
         :rtype: str
         """
-        return dedent(value)
+        return dedent(value.lstrip("\n"))
 
     @field_validator("stages", mode="after")
     def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
@@ -424,11 +430,14 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage {stage_id!r} does not exists in this job.")
 
-    def check_needs(self, jobs: dict[str, Any]) -> Status:  # pragma: no cov
+    def check_needs(
+        self, jobs: dict[str, DictData]
+    ) -> Status:  # pragma: no cov
         """Return trigger status from checking job's need trigger rule logic was
         valid. The return status should be SUCCESS, FAILED, WAIT, or SKIP.
 
-        :param jobs: A mapping of job ID and its context data.
+        :param jobs: (dict[str, DictData]) A mapping of job ID and its context
+            data that return from execution process.
 
         :raise NotImplementedError: If the job trigger rule out of scope.
 
@@ -445,28 +454,34 @@ class Job(BaseModel):
         }
         if len(need_exist) != len(self.needs):
             return WAIT
-        elif all("skipped" in need_exist[job] for job in need_exist):
+        elif all(need_exist[job].get("skipped", False) for job in need_exist):
             return SKIP
         elif self.trigger_rule == Rule.ALL_DONE:
             return SUCCESS
         elif self.trigger_rule == Rule.ALL_SUCCESS:
             rs = all(
-                k not in need_exist[job]
-                for k in ("errors", "skipped")
+                (
+                    "errors" not in need_exist[job]
+                    and not need_exist[job].get("skipped", False)
+                )
                 for job in need_exist
             )
         elif self.trigger_rule == Rule.ALL_FAILED:
             rs = all("errors" in need_exist[job] for job in need_exist)
         elif self.trigger_rule == Rule.ONE_SUCCESS:
             rs = sum(
-                k not in need_exist[job]
-                for k in ("errors", "skipped")
+                (
+                    "errors" not in need_exist[job]
+                    and not need_exist[job].get("skipped", False)
+                )
                 for job in need_exist
             ) + 1 == len(self.needs)
         elif self.trigger_rule == Rule.ONE_FAILED:
             rs = sum("errors" in need_exist[job] for job in need_exist) == 1
         elif self.trigger_rule == Rule.NONE_SKIPPED:
-            rs = all("skipped" not in need_exist[job] for job in need_exist)
+            rs = all(
+                not need_exist[job].get("skipped", False) for job in need_exist
+            )
         elif self.trigger_rule == Rule.NONE_FAILED:
             rs = all("errors" not in need_exist[job] for job in need_exist)
         else:  # pragma: no cov
@@ -608,20 +623,19 @@ class Job(BaseModel):
         :param event: (Event) An Event manager instance that use to cancel this
             execution if it forces stopped by parent execution.
 
-        :raise NotImplementedError: If the `runs-on` value does not implement on
-            this execution.
-
         :rtype: Result
         """
         result: Result = Result.construct_with_rs_or_id(
             run_id=run_id,
             parent_run_id=parent_run_id,
-            id_logic=(self.id or "not-set"),
+            id_logic=(self.id or "EMPTY"),
             extras=self.extras,
         )
 
         result.trace.info(
-            f"[JOB]: Execute: {self.id!r} on {self.runs_on.type.value!r}"
+            f"[JOB]: Execute "
+            f"{''.join(self.runs_on.type.value.split('_')).title()}: "
+            f"{self.id!r}"
         )
         if self.runs_on.type == RunsOn.LOCAL:
             return local_execute(
@@ -642,12 +656,18 @@ class Job(BaseModel):
                 event=event,
             )
 
-        # pragma: no cov
         result.trace.error(
-            f"[JOB]: Execute not support runs-on: {self.runs_on.type!r} yet."
+            f"[JOB]: Execute not support runs-on: {self.runs_on.type.value!r} "
+            f"yet."
         )
-        raise NotImplementedError(
-            f"Execute runs-on type: {self.runs_on.type} does not support yet."
+        return result.catch(
+            status=FAILED,
+            context={
+                "errors": JobException(
+                    f"Execute runs-on type: {self.runs_on.type.value!r} does "
+                    f"not support yet."
+                ).to_dict(),
+            },
         )
 
 
@@ -659,10 +679,10 @@ def local_execute_strategy(
     result: Result | None = None,
     event: Event | None = None,
 ) -> Result:
-    """Local job strategy execution with passing dynamic parameters from the
-    workflow execution to strategy matrix.
+    """Local strategy execution with passing dynamic parameters from the
+    job execution and strategy matrix.
 
-        This execution is the minimum level of execution of this job model.
+        This execution is the minimum level of job execution.
     It different with `self.execute` because this method run only one
     strategy and return with context of this strategy data.
 
@@ -679,22 +699,22 @@ def local_execute_strategy(
     :param event: (Event) An Event manager instance that use to cancel this
         execution if it forces stopped by parent execution.
 
-    :raise JobException: If stage execution raise any error as `StageException`
-        or `UtilException`.
+    :raise JobException: If event was set.
+    :raise JobException: If stage execution raise any error as `StageException`.
+    :raise JobException: If the result from execution has `FAILED` status.
 
     :rtype: Result
     """
     result: Result = result or Result(
-        run_id=gen_id(job.id or "not-set", unique=True),
+        run_id=gen_id(job.id or "EMPTY", unique=True),
         extras=job.extras,
     )
     if strategy:
         strategy_id: str = gen_id(strategy)
-        result.trace.info(f"[JOB]: Start Strategy: {strategy_id!r}")
+        result.trace.info(f"[JOB]: Execute Strategy: {strategy_id!r}")
         result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
     else:
         strategy_id: str = "EMPTY"
-        result.trace.info("[JOB]: Start Strategy: 'EMPTY'")
 
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
@@ -709,11 +729,8 @@ def local_execute_strategy(
             continue
 
         if event and event.is_set():
-            error_msg: str = (
-                "Job strategy was canceled from event that had set before "
-                "job strategy execution."
-            )
-            return result.catch(
+            error_msg: str = "Job strategy was canceled because event was set."
+            result.catch(
                 status=CANCEL,
                 context={
                     strategy_id: {
@@ -723,6 +740,7 @@ def local_execute_strategy(
                     },
                 },
             )
+            raise JobException(error_msg, refs=strategy_id)
 
         try:
             result.trace.info(f"[JOB]: Execute Stage: {stage.iden!r}")
@@ -733,8 +751,7 @@ def local_execute_strategy(
                 event=event,
             )
             stage.set_outputs(rs.context, to=context)
-        except (StageException, UtilException) as e:
-            result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
+        except StageException as e:
             result.catch(
                 status=FAILED,
                 context={
@@ -746,15 +763,15 @@ def local_execute_strategy(
                 },
             )
             raise JobException(
-                f"Stage raise: {e.__class__.__name__}: {e}"
+                message=f"Handler Error: {e.__class__.__name__}: {e}",
+                refs=strategy_id,
             ) from e
 
         if rs.status == FAILED:
             error_msg: str = (
-                f"Strategy break because stage, {stage.iden!r}, return FAILED "
-                f"status."
+                f"Strategy break because stage, {stage.iden!r}, return "
+                f"`FAILED` status."
             )
-            result.trace.warning(f"[JOB]: {error_msg}")
             result.catch(
                 status=FAILED,
                 context={
@@ -765,7 +782,7 @@ def local_execute_strategy(
                     },
                 },
             )
-            raise JobException(error_msg)
+            raise JobException(error_msg, refs=strategy_id)
 
     return result.catch(
         status=SUCCESS,
@@ -787,11 +804,19 @@ def local_execute(
     event: Event | None = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
-    execution or itself execution. It will generate matrix values at the first
+    execution or directly. It will generate matrix values at the first
     step and run multithread on this metrics to the `stages` field of this job.
 
-        This method does not raise any `JobException` if it runs with
-    multi-threading strategy.
+    Important:
+        This method does not raise any `JobException` because it allows run
+    parallel mode. If it raises error from strategy execution, it will catch
+    that error and store it in the `errors` key with list of error.
+
+        {
+            "errors": [
+                {"name": "...", "message": "..."}, ...
+            ]
+        }
 
     :param job: (Job) A job model.
     :param params: (DictData) A parameter data.
@@ -805,20 +830,20 @@ def local_execute(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
 
-    event: Event = Event() if event is None else event
+    event: Event = event or Event()
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
     workers: int = job.strategy.max_parallel
     result.trace.info(
-        f"[JOB]: {ls}-Execute: {job.id} with {workers} "
+        f"[JOB]: Execute {ls}: {job.id} with {workers} "
         f"worker{'s' if workers > 1 else ''}."
     )
 
-    if event and event.is_set():  # pragma: no cov
+    if event and event.is_set():
         return result.catch(
             status=CANCEL,
             context={
@@ -854,14 +879,16 @@ def local_execute(
             done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
             if len(done) != len(futures):
                 result.trace.warning(
-                    "[JOB]: Set event for stop pending stage future."
+                    "[JOB]: Handler Fail-Fast: Got exception and set event."
                 )
                 event.set()
                 for future in not_done:
                     future.cancel()
+                time.sleep(0.075)
 
             nd: str = f", strategies not run: {not_done}" if not_done else ""
-            result.trace.debug(f"... Strategy set Fail-Fast{nd}")
+            result.trace.debug(f"[JOB]: ... Job was set Fail-Fast{nd}")
+            done: list[Future] = as_completed(futures)
 
         for future in done:
             try:
@@ -869,12 +896,14 @@ def local_execute(
             except JobException as e:
                 status = FAILED
                 result.trace.error(
-                    f"[JOB]: {ls}: {e.__class__.__name__}:{NEWLINE}{e}"
+                    f"[JOB]: {ls} Error Handler:||{e.__class__.__name__}:||{e}"
                 )
                 if "errors" in context:
-                    context["errors"].append(e.to_dict())
+                    context["errors"][e.refs] = e.to_dict()
                 else:
-                    context["errors"] = [e.to_dict()]
+                    context["errors"] = e.to_dict(with_refs=True)
+            except CancelledError:
+                pass
     return result.catch(status=status, context=context)
 
 
@@ -902,7 +931,7 @@ def self_hosted_execute(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
 
@@ -948,7 +977,7 @@ def azure_batch_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-) -> Result:  # pragma no cov
+) -> Result:  # pragma: no cov
     """Azure Batch job execution that will run all job's stages on the Azure
     Batch Node and extract the result file to be returning context result.
 
@@ -978,7 +1007,7 @@ def azure_batch_execute(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
     if event and event.is_set():
@@ -1002,7 +1031,7 @@ def docker_execution(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-):
+):  # pragma: no cov
     """Docker job execution.
 
     Steps:
@@ -1013,7 +1042,7 @@ def docker_execution(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
     if event and event.is_set():

ddeutil/workflow/logs.py

@@ -29,7 +29,7 @@ from typing_extensions import Self
 
 from .__types import DictData, DictStr
 from .conf import config, dynamic
-from .utils import cut_id, get_dt_now
+from .utils import cut_id, get_dt_now, prepare_newline
 
 
 @lru_cache
@@ -71,7 +71,9 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
 
 
 class TraceMeta(BaseModel):  # pragma: no cov
-    """Trace Meta model."""
+    """Trace Metadata model for making the current metadata of this CPU, Memory
+    process, and thread data.
+    """
 
     mode: Literal["stdout", "stderr"]
     datetime: str
@@ -91,6 +93,11 @@ class TraceMeta(BaseModel):  # pragma: no cov
     ) -> Self:
         """Make the current TraceMeta instance that catching local state.
 
+        :param mode: A metadata mode.
+        :param message: A message.
+        :param extras: (DictData) An extra parameter that want to override core
+            config values.
+
         :rtype: Self
         """
         frame_info: Traceback = getframeinfo(
@@ -232,7 +239,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
+        msg: str = prepare_newline(self.make_message(message))
 
         if mode != "debug" or (
             mode == "debug" and dynamic("debug", extras=self.extras)
@@ -445,6 +452,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
     async def awriter(
         self, message: str, is_err: bool = False
     ) -> None:  # pragma: no cov
+        """Write with async mode."""
         if not dynamic("enable_write_log", extras=self.extras):
             return
 
@@ -744,7 +752,7 @@ class FileAudit(BaseAudit):
 
         # NOTE: Check environ variable was set for real writing.
         if not dynamic("enable_write_audit", extras=self.extras):
-            trace.debug("[LOG]: Skip writing log cause config was set")
+            trace.debug("[AUDIT]: Skip writing log cause config was set")
             return self
 
         log_file: Path = (
@@ -813,7 +821,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
 
         # NOTE: Check environ variable was set for real writing.
         if not dynamic("enable_write_audit", extras=self.extras):
-            trace.debug("[LOG]: Skip writing log cause config was set")
+            trace.debug("[AUDIT]: Skip writing log cause config was set")
             return self
 
         raise NotImplementedError("SQLiteAudit does not implement yet.")

ddeutil/workflow/result.py

@@ -37,6 +37,14 @@ class Status(IntEnum):
     SKIP: int = 3
     CANCEL: int = 4
 
+    @property
+    def emoji(self) -> str:
+        """Return the emoji value of this status.
+
+        :rtype: str
+        """
+        return {0: "✅", 1: "❌", 2: "🟡", 3: "⏩", 4: "🚫"}[self.value]
+
 
 SUCCESS = Status.SUCCESS
 FAILED = Status.FAILED
@@ -46,10 +54,7 @@ CANCEL = Status.CANCEL
 
 
 @dataclass(
-    config=ConfigDict(
-        arbitrary_types_allowed=True,
-        use_enum_values=True,
-    ),
+    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True),
 )
 class Result:
     """Result Pydantic Model for passing and receiving data context from any