ddeutil-workflow 0.0.85__py3-none-any.whl → 0.0.86__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1,2 +1,2 @@
-__version__: str = "0.0.85"
+__version__: str = "0.0.86"
 __python_version__: str = "3.9"

ddeutil/workflow/api/routes/job.py

@@ -10,6 +10,7 @@ from typing import Any, Optional
 
 from fastapi import APIRouter, Body
 from fastapi import status as st
+from fastapi.encoders import jsonable_encoder
 from fastapi.responses import UJSONResponse
 
 from ...__types import DictData
@@ -66,7 +67,7 @@ async def job_execute(
                     exclude_unset=True,
                 ),
                 "params": params,
-                "context": context,
+                "context": jsonable_encoder(context),
             },
             status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
         )
@@ -81,7 +82,7 @@ async def job_execute(
                 exclude_unset=True,
             ),
             "params": params,
-            "context": context,
+            "context": jsonable_encoder(context),
         },
         status_code=st.HTTP_200_OK,
     )

ddeutil/workflow/conf.py

@@ -117,7 +117,7 @@ class Config:  # pragma: no cov
     def debug(self) -> bool:
         """Debug flag for echo log that use DEBUG mode.
 
-        :rtype: bool
+        Returns: bool
         """
         return str2bool(env("LOG_DEBUG_MODE", "true"))
 
@@ -472,9 +472,11 @@ def dynamic(
 def pass_env(value: T) -> T:  # pragma: no cov
     """Passing environment variable to an input value.
 
-    :param value: (Any) A value that want to pass env var searching.
+    Args:
+        value (Any): A value that want to pass env var searching.
 
-    :rtype: Any
+    Returns:
+        Any: An any value that have passed environment variable.
     """
     if isinstance(value, dict):
         return {k: pass_env(value[k]) for k in value}

ddeutil/workflow/errors.py

@@ -208,4 +208,7 @@ class WorkflowCancelError(WorkflowError): ...
 class WorkflowTimeoutError(WorkflowError): ...
 
 
+class WorkflowSkipError(WorkflowError): ...
+
+
 class ParamError(WorkflowError): ...

ddeutil/workflow/job.py

@@ -75,7 +75,7 @@ from .result import (
 from .reusables import has_template, param2template
 from .stages import Stage
 from .traces import Trace, get_trace
-from .utils import cross_product, extract_id, filter_func, gen_id
+from .utils import cross_product, extract_id, filter_func, gen_id, get_dt_now
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -824,9 +824,14 @@ class Job(BaseModel):
         status: dict[str, Status] = (
             {"status": output.pop("status")} if "status" in output else {}
         )
+        info: DictData = (
+            {"info": output.pop("info")} if "info" in output else {}
+        )
         kwargs: DictData = kwargs or {}
         if self.strategy.is_set():
-            to["jobs"][_id] = {"strategies": output} | errors | status | kwargs
+            to["jobs"][_id] = (
+                {"strategies": output} | errors | status | info | kwargs
+            )
         elif len(k := output.keys()) > 1:  # pragma: no cov
             raise JobError(
                 "Strategy output from execution return more than one ID while "
@@ -835,7 +840,7 @@ class Job(BaseModel):
         else:
             _output: DictData = {} if len(k) == 0 else output[list(k)[0]]
             _output.pop("matrix", {})
-            to["jobs"][_id] = _output | errors | status | kwargs
+            to["jobs"][_id] = _output | errors | status | info | kwargs
         return to
 
     def get_outputs(
@@ -851,7 +856,8 @@ class Job(BaseModel):
             output (DictData): A job outputs data that want to extract
             job_id (StrOrNone): A job ID if the `id` field does not set.
 
-        :rtype: DictData
+        Returns:
+            DictData: An output data.
         """
         _id: str = self.id or job_id
         if self.strategy.is_set():
@@ -878,17 +884,31 @@ class Job(BaseModel):
         params: DictData,
         run_id: str,
         context: DictData,
+        *,
         parent_run_id: Optional[str] = None,
         event: Optional[Event] = None,
     ) -> Result:
         """Process routing method that will route the provider function depend
         on runs-on value.
+
+        Args:
+            params (DictData): A parameter data that want to use in this
+                execution.
+            run_id (str): A running stage ID.
+            context (DictData): A context data that was passed from handler
+                method.
+            parent_run_id (str, default None): A parent running ID.
+            event (Event, default None): An event manager that use to track
+                parent process was not force stopped.
+
+        Returns:
+            Result: The execution result with status and context data.
         """
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         trace.info(
-            f"[JOB]: Routing for "
+            f"[JOB]: Routing "
             f"{''.join(self.runs_on.type.value.split('_')).title()}: "
             f"{self.id!r}"
         )
@@ -946,6 +966,7 @@ class Job(BaseModel):
                 run_id=parent_run_id,
                 event=event,
             )
+
         if rs is None:
             trace.error(
                 f"[JOB]: Execution not support runs-on: {self.runs_on.type.value!r} "
@@ -964,20 +985,20 @@ class Job(BaseModel):
                 },
                 extras=self.extras,
             )
-        if rs.status in (CANCEL, SKIP):
-            trace.debug(
-                f"[JOB]: Job process routing got result status be {rs.status}"
-            )
+
+        if rs.status == SKIP:
+            raise JobSkipError("Job got skipped status.")
+        elif rs.status == CANCEL:
+            raise JobCancelError("Job got canceled status.")
         elif rs.status == FAILED:
-            raise JobError("[JOB]: Job process error")
+            raise JobError("Job process error")
         return rs
 
     def _execute(
         self,
         params: DictData,
-        run_id: str,
         context: DictData,
-        parent_run_id: Optional[str] = None,
+        trace: Trace,
         event: Optional[Event] = None,
     ) -> Result:
         """Wrapped the route execute method before returning to handler
@@ -988,45 +1009,40 @@ class Job(BaseModel):
 
         Args:
             params: A parameter data that want to use in this execution
-            run_id:
             context:
-            parent_run_id:
-            event:
+            trace (Trace):
+            event (Event, default None):
 
         Returns:
             Result: The wrapped execution result.
         """
         current_retry: int = 0
+        maximum_retry: int = self.retry + 1
         exception: Exception
         catch(context, status=WAIT)
-        trace: Trace = get_trace(
-            run_id, parent_run_id=parent_run_id, extras=self.extras
-        )
         try:
             return self.process(
                 params,
-                run_id,
+                run_id=trace.run_id,
                 context=context,
-                parent_run_id=parent_run_id,
+                parent_run_id=trace.parent_run_id,
                 event=event,
             )
         except (JobCancelError, JobSkipError):
             trace.debug("[JOB]: process raise skip or cancel error.")
             raise
         except Exception as e:
+            if self.retry == 0:
+                raise
+
             current_retry += 1
             exception = e
-        finally:
-            trace.debug("[JOB]: Failed at the first execution.")
-
-        if self.retry == 0:
-            raise exception
 
         trace.warning(
-            f"[JOB]: Retry count: {current_retry} ... "
+            f"[JOB]: Retry count: {current_retry}/{maximum_retry} ... "
             f"( {exception.__class__.__name__} )"
         )
-        while current_retry < (self.retry + 1):
+        while current_retry < maximum_retry:
             try:
                 catch(
                     context=context,
@@ -1035,18 +1051,18 @@ class Job(BaseModel):
                 )
                 return self.process(
                     params,
-                    run_id,
+                    run_id=trace.run_id,
                     context=context,
-                    parent_run_id=parent_run_id,
+                    parent_run_id=trace.parent_run_id,
                     event=event,
                 )
             except (JobCancelError, JobSkipError):
                 trace.debug("[JOB]: process raise skip or cancel error.")
                 raise
-            except exception as e:
+            except Exception as e:
                 current_retry += 1
                 trace.warning(
-                    f"[JOB]: Retry count: {current_retry} ... "
+                    f"[JOB]: Retry count: {current_retry}/{maximum_retry} ... "
                     f"( {e.__class__.__name__} )"
                 )
                 exception = e
@@ -1083,32 +1099,40 @@ class Job(BaseModel):
         parent_run_id, run_id = extract_id(
             (self.id or "EMPTY"), run_id=run_id, extras=self.extras
         )
+        context: DictData = {
+            "status": WAIT,
+            "info": {"exec_start": get_dt_now()},
+        }
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
-        context: DictData = {"status": WAIT}
         try:
             trace.info(
                 f"[JOB]: Handler {self.runs_on.type.name}: "
                 f"{(self.id or 'EMPTY')!r}."
             )
-            result_caught: Result = self._execute(
+            result: Result = self._execute(
                 params,
-                run_id=run_id,
                 context=context,
-                parent_run_id=parent_run_id,
+                trace=trace,
                 event=event,
             )
-            return result_caught.make_info(
-                {"execution_time": time.monotonic() - ts}
-            )
-        except JobError:  # pragma: no cov
+            return result
+        except JobError as e:  # pragma: no cov
+            if isinstance(e, JobSkipError):
+                trace.error(f"[JOB]: ⏭️ Skip: {e}")
+
+            st: Status = get_status_from_error(e)
             return Result.from_trace(trace).catch(
-                status=FAILED,
-                context=catch(context, status=FAILED),
-                info={"execution_time": time.monotonic() - ts},
+                status=st, context=catch(context, status=st)
             )
         finally:
+            context["info"].update(
+                {
+                    "exec_end": get_dt_now(),
+                    "exec_latency": round(time.monotonic() - ts, 6),
+                }
+            )
             trace.debug("[JOB]: End Handler job execution.")
 
 

ddeutil/workflow/result.py

@@ -12,30 +12,29 @@ states and the Result dataclass for context transfer between workflow components
 from __future__ import annotations
 
 from dataclasses import field
+from datetime import datetime
 from enum import Enum
 from typing import Any, Optional, TypedDict, Union
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
-from pydantic.functional_validators import model_validator
 from typing_extensions import NotRequired, Self
 
-from . import (
+from .__types import DictData
+from .errors import (
+    BaseError,
+    ErrorData,
     JobCancelError,
-    JobError,
     JobSkipError,
+    ResultError,
     StageCancelError,
-    StageError,
     StageNestedCancelError,
-    StageNestedError,
     StageNestedSkipError,
     StageSkipError,
     WorkflowCancelError,
-    WorkflowError,
+    WorkflowSkipError,
 )
-from .__types import DictData
-from .audits import Trace, get_trace
-from .errors import ErrorData, ResultError
+from .traces import Trace, get_trace
 from .utils import default_gen_id
 
 
@@ -110,11 +109,11 @@ def validate_statuses(statuses: list[Status]) -> Status:
         Status: Final consolidated status based on workflow logic
 
     Example:
-        >>> # Mixed statuses - FAILED takes priority
+        Case: Mixed statuses - FAILED takes priority
         >>> validate_statuses([SUCCESS, FAILED, SUCCESS])
         >>> # Returns: FAILED
 
-        >>> # All same status
+        Case: All same status
         >>> validate_statuses([SUCCESS, SUCCESS, SUCCESS])
         >>> # Returns: SUCCESS
     """
@@ -131,21 +130,7 @@ def validate_statuses(statuses: list[Status]) -> Status:
 
 
 def get_status_from_error(
-    error: Union[
-        StageError,
-        StageCancelError,
-        StageSkipError,
-        StageNestedCancelError,
-        StageNestedError,
-        StageNestedSkipError,
-        JobError,
-        JobCancelError,
-        JobSkipError,
-        WorkflowError,
-        WorkflowCancelError,
-        Exception,
-        BaseException,
-    ]
+    error: Union[BaseError, Exception, BaseException]
 ) -> Status:
     """Get the Status from the error object.
 
@@ -155,7 +140,10 @@ def get_status_from_error(
     Returns:
         Status: The status from the specific exception class.
     """
-    if isinstance(error, (StageNestedSkipError, StageSkipError, JobSkipError)):
+    if isinstance(
+        error,
+        (StageNestedSkipError, StageSkipError, JobSkipError, WorkflowSkipError),
+    ):
         return SKIP
     elif isinstance(
         error,
@@ -189,25 +177,8 @@ class Result:
     extras: DictData = field(default_factory=dict, compare=False, repr=False)
     status: Status = field(default=WAIT)
     context: Optional[DictData] = field(default=None)
-    info: DictData = field(default_factory=dict)
     run_id: str = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None)
-    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
-
-    @model_validator(mode="after")
-    def __prepare_trace(self) -> Self:
-        """Prepare trace field that want to pass after its initialize step.
-
-        :rtype: Self
-        """
-        if self.trace is None:  # pragma: no cov
-            self.trace: Trace = get_trace(
-                self.run_id,
-                parent_run_id=self.parent_run_id,
-                extras=self.extras,
-            )
-
-        return self
 
     @classmethod
     def from_trace(cls, trace: Trace):
@@ -216,7 +187,13 @@ class Result:
             run_id=trace.run_id,
             parent_run_id=trace.parent_run_id,
             extras=trace.extras,
-            trace=trace,
+        )
+
+    def gen_trace(self) -> Trace:
+        return get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
         )
 
     def catch(
@@ -251,18 +228,16 @@ class Result:
                     self.__dict__["context"][k].update(kwargs[k])
                 # NOTE: Exclude the `info` key for update information data.
                 elif k == "info":
-                    self.__dict__["info"].update(kwargs["info"])
+                    if "info" in self.__dict__["context"]:
+                        self.__dict__["context"].update(kwargs[k])
+                    else:
+                        self.__dict__["context"]["info"] = kwargs[k]
                 else:
                     raise ResultError(
                         f"The key {k!r} does not exists on context data."
                     )
         return self
 
-    def make_info(self, data: DictData) -> Self:
-        """Making information."""
-        self.__dict__["info"].update(data)
-        return self
-
 
 def catch(
     context: DictData,
@@ -292,19 +267,35 @@ def catch(
             context[k].update(kwargs[k])
         # NOTE: Exclude the `info` key for update information data.
         elif k == "info":
-            context.update({"info": kwargs["info"]})
-        else:
-            raise ResultError(f"The key {k!r} does not exists on context data.")
+            if "info" in context:
+                context.update(kwargs[k])
+            else:
+                context["info"] = kwargs[k]
+        # ARCHIVE:
+        # else:
+        #     raise ResultError(f"The key {k!r} does not exist on context data.")
     return context
 
 
+class Info(TypedDict):
+    exec_start: datetime
+    exec_end: NotRequired[datetime]
+    exec_latency: NotRequired[float]
+
+
+class System(TypedDict):
+    __sys_release_dryrun_mode: NotRequired[bool]
+    __sys_exec_break_circle: NotRequired[str]
+
+
 class Context(TypedDict):
     """Context dict typed."""
 
     status: Status
+    info: Info
+    sys: NotRequired[System]
     context: NotRequired[DictData]
     errors: NotRequired[Union[list[ErrorData], ErrorData]]
-    info: NotRequired[DictData]
 
 
 class Layer(str, Enum):