ddeutil-workflow 0.0.85__py3-none-any.whl → 0.0.86__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -122,6 +122,7 @@ from .utils import (
     extract_id,
     filter_func,
     gen_id,
+    get_dt_now,
     make_exec,
     to_train,
 )
@@ -133,9 +134,9 @@ DictOrModel = Union[DictData, BaseModel]
 class BaseStage(BaseModel, ABC):
     """Abstract base class for all stage implementations.
 
-    BaseStage provides the foundation for all stage types in the workflow system.
-    It defines the common interface and metadata fields that all stages must
-    implement, ensuring consistent behavior across different stage types.
+        BaseStage provides the foundation for all stage types in the workflow
+    system. It defines the common interface and metadata fields that all stages
+    must implement, ensuring consistent behavior across different stage types.
 
     This abstract class handles core stage functionality including:
         - Stage identification and naming
@@ -143,7 +144,7 @@ class BaseStage(BaseModel, ABC):
         - Output management and templating
         - Execution lifecycle management
 
-    Custom stages should inherit from this class and implement the abstract
+        Custom stages should inherit from this class and implement the abstract
     `process()` method to define their specific execution behavior.
 
     Attributes:
@@ -189,7 +190,7 @@ class BaseStage(BaseModel, ABC):
         default=None,
         description=(
             "A stage condition statement to allow stage executable. This field "
-            "alise with `if` field."
+            "alias with `if` key."
         ),
         alias="if",
     )
@@ -260,6 +261,13 @@ class BaseStage(BaseModel, ABC):
         """Process abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
 
+            For process method, it designs to break process with any status by
+        raise it with a specific exception class.
+
+            - StageError            -> FAILED
+            - StageSkipError        -> SKIP
+            - StageCancelError      -> CANCEL
+
         Args:
             params (DictData): A parameter data that want to use in this
                 execution.
@@ -321,13 +329,16 @@ class BaseStage(BaseModel, ABC):
         parent_run_id, run_id = extract_id(
             self.iden, run_id=run_id, extras=self.extras
         )
-        context: DictData = {"status": WAIT}
+        context: DictData = {
+            "status": WAIT,
+            "info": {"exec_start": get_dt_now()},
+        }
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         try:
             _id: str = (
-                f" with ID: {param2template(self.id, params=params)!r}"
+                f" with ID: {self.pass_template(self.id, params=params)!r}"
                 if self.id
                 else ""
             )
@@ -348,84 +359,73 @@ class BaseStage(BaseModel, ABC):
 
             # NOTE: Start call wrapped execution method that will use custom
             #   execution before the real execution from inherit stage model.
-            result_caught: Result = self._execute(
+            result: Result = self._execute(
                 params,
-                run_id=run_id,
                 context=context,
-                parent_run_id=parent_run_id,
+                trace=trace,
                 event=event,
             )
-            if result_caught.status == WAIT:  # pragma: no cov
+            if result.status == WAIT:  # pragma: no cov
                 raise StageError(
                     "Status from execution should not return waiting status."
                 )
-            return result_caught.make_info(
-                {"execution_time": time.monotonic() - ts}
-            )
+            return result
 
         # NOTE: Catch this error in this line because the execution can raise
         #   this exception class at other location.
-        except (
-            StageSkipError,
-            StageNestedSkipError,
-            StageNestedError,
-            StageError,
-        ) as e:  # pragma: no cov
+        except StageError as e:  # pragma: no cov
             updated: Optional[DictData] = {"errors": e.to_dict()}
             if isinstance(e, StageNestedError):
-                trace.error(f"[STAGE]: Nested: {e}")
+                trace.error(f"[STAGE]: ⚠️ Nested: {e}")
             elif isinstance(e, (StageSkipError, StageNestedSkipError)):
-                trace.error(f"[STAGE]: ⏭️ Skip: {e}")
+                trace.error(f"[STAGE]: ⏭️ Skip: {e}", module="stage")
                 updated = None
             elif e.allow_traceback:
                 trace.error(
-                    f"[STAGE]: Stage Failed:||🚨 {traceback.format_exc()}||"
+                    f"[STAGE]: 📢 Stage Failed:||🚨 {traceback.format_exc()}||"
                 )
             else:
                 trace.error(
                     f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
                 )
             st: Status = get_status_from_error(e)
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=st,
                 context=catch(context, status=st, updated=updated),
-                info={"execution_time": time.monotonic() - ts},
-                extras=self.extras,
             )
         except Exception as e:
             trace.error(
-                f"[STAGE]: Error Failed:||🚨 {traceback.format_exc()}||"
+                f"💥 Error Failed:||🚨 {traceback.format_exc()}||",
+                module="stage",
             )
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=FAILED,
                 context=catch(
                     context, status=FAILED, updated={"errors": to_dict(e)}
                 ),
-                info={"execution_time": time.monotonic() - ts},
-                extras=self.extras,
             )
         finally:
-            trace.debug("[STAGE]: End Handler stage execution.")
+            context["info"].update(
+                {
+                    "exec_end": get_dt_now(),
+                    "exec_latency": round(time.monotonic() - ts, 6),
+                }
+            )
+            trace.debug("End Handler stage execution.", module="stage")
 
     def _execute(
         self,
         params: DictData,
-        run_id: str,
         context: DictData,
-        parent_run_id: Optional[str] = None,
+        trace: Trace,
         event: Optional[Event] = None,
     ) -> Result:
         """Wrapped the process method before returning to handler execution.
 
         Args:
             params: A parameter data that want to use in this execution.
-            run_id (str):
             context:
-            parent_run_id:
+            trace (Trace):
             event: An event manager that use to track parent process
                 was not force stopped.
 
@@ -435,9 +435,9 @@ class BaseStage(BaseModel, ABC):
         catch(context, status=WAIT)
         return self.process(
             params,
-            run_id=run_id,
+            run_id=trace.run_id,
             context=context,
-            parent_run_id=parent_run_id,
+            parent_run_id=trace.parent_run_id,
             event=event,
         )
 
@@ -454,7 +454,7 @@ class BaseStage(BaseModel, ABC):
             For example of setting output method, If you receive process output
         and want to set on the `to` like;
 
-            ... (i)   output: {'foo': 'bar', 'skipped': True}
+            ... (i)   output: {'foo': 'bar', 'status': SUCCESS, 'info': {}}
             ... (ii)  to: {'stages': {}}
 
             The received context in the `to` argument will be;
@@ -463,7 +463,8 @@ class BaseStage(BaseModel, ABC):
                         'stages': {
                             '<stage-id>': {
                                 'outputs': {'foo': 'bar'},
-                                'skipped': True,
+                                'status': SUCCESS,
+                                'info': {},
                             }
                         }
                     }
@@ -504,8 +505,13 @@ class BaseStage(BaseModel, ABC):
         status: dict[str, Status] = (
             {"status": output.pop("status")} if "status" in output else {}
         )
+        info: DictData = (
+            {"info": output.pop("info")} if "info" in output else {}
+        )
         kwargs: DictData = kwargs or {}
-        to["stages"][_id] = {"outputs": output} | errors | status | kwargs
+        to["stages"][_id] = (
+            {"outputs": output} | errors | status | info | kwargs
+        )
         return to
 
     def get_outputs(self, output: DictData) -> DictData:
@@ -533,15 +539,18 @@ class BaseStage(BaseModel, ABC):
         """Return true if condition of this stage do not correct. This process
         use build-in eval function to execute the if-condition.
 
-        :param params: (DictData) A parameters that want to pass to condition
-            template.
+        Args:
+            params (DictData): A parameters that want to pass to condition
+                template.
 
-        :raise StageError: When it has any error raise from the eval
-            condition statement.
-        :raise StageError: When return type of the eval condition statement
-            does not return with boolean type.
+        Raises:
+            StageError: When it has any error raise from the eval
+                condition statement.
+            StageError: When return type of the eval condition statement
+                does not return with boolean type.
 
-        :rtype: bool
+        Returns:
+            bool: True if the condition is valid with the current parameters.
         """
         # NOTE: Support for condition value is empty string.
         if not self.condition:
@@ -569,9 +578,11 @@ class BaseStage(BaseModel, ABC):
         """Generate stage ID that dynamic use stage's name if it ID does not
         set.
 
-        :param params: (DictData) A parameter or context data.
+        Args:
+            params (DictData): A parameter or context data.
 
-        :rtype: str
+        Returns:
+            str: An ID that already generated from id or name fields.
         """
         return (
             param2template(self.id, params=params, extras=self.extras)
@@ -779,15 +790,19 @@ class BaseAsyncStage(BaseStage, ABC):
             Result: The execution result with status and context data.
         """
         ts: float = time.monotonic()
-        parent_run_id: StrOrNone = run_id
-        run_id: str = gen_id(self.iden, unique=True, extras=self.extras)
-        context: DictData = {}
+        parent_run_id, run_id = extract_id(
+            self.iden, run_id=run_id, extras=self.extras
+        )
+        context: DictData = {
+            "status": WAIT,
+            "info": {"exec_start": get_dt_now()},
+        }
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         try:
             _id: str = (
-                f" with ID: {param2template(self.id, params=params)!r}"
+                f" with ID: {self.pass_template(self.id, params=params)!r}"
                 if self.id
                 else ""
             )
@@ -808,67 +823,59 @@ class BaseAsyncStage(BaseStage, ABC):
 
             # NOTE: Start call wrapped execution method that will use custom
             #   execution before the real execution from inherit stage model.
-            result_caught: Result = await self._axecute(
+            result: Result = await self._axecute(
                 params,
                 run_id=run_id,
                 context=context,
                 parent_run_id=parent_run_id,
                 event=event,
             )
-            if result_caught.status == WAIT:  # pragma: no cov
+            if result.status == WAIT:  # pragma: no cov
                 raise StageError(
                     "Status from execution should not return waiting status."
                 )
-            return result_caught
+            return result
 
         # NOTE: Catch this error in this line because the execution can raise
         #   this exception class at other location.
-        except (
-            StageSkipError,
-            StageNestedSkipError,
-            StageNestedError,
-            StageError,
-        ) as e:  # pragma: no cov
+        except StageError as e:  # pragma: no cov
+            updated: Optional[DictData] = {"errors": e.to_dict()}
             if isinstance(e, StageNestedError):
-                await trace.ainfo(f"[STAGE]: Nested: {e}")
+                await trace.aerror(f"[STAGE]: ⚠️ Nested: {e}")
             elif isinstance(e, (StageSkipError, StageNestedSkipError)):
-                await trace.ainfo(f"[STAGE]: ⏭️ Skip: {e}")
+                await trace.aerror(f"[STAGE]: ⏭️ Skip: {e}")
+                updated = None
+            elif e.allow_traceback:
+                await trace.aerror(
+                    f"[STAGE]: 📢 Stage Failed:||🚨 {traceback.format_exc()}||"
+                )
             else:
-                await trace.ainfo(
-                    f"[STAGE]: Stage Failed:||🚨 {traceback.format_exc()}||"
+                await trace.aerror(
+                    f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
                 )
             st: Status = get_status_from_error(e)
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=st,
-                context=catch(
-                    context,
-                    status=st,
-                    updated=(
-                        None
-                        if isinstance(e, (StageSkipError, StageNestedSkipError))
-                        else {"errors": e.to_dict()}
-                    ),
-                ),
-                info={"execution_time": time.monotonic() - ts},
-                extras=self.extras,
+                context=catch(context, status=st, updated=updated),
             )
         except Exception as e:
             await trace.aerror(
-                f"[STAGE]: Error Failed:||🚨 {traceback.format_exc()}||"
+                f"💥 Error Failed:||🚨 {traceback.format_exc()}||",
+                module="stage",
             )
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=FAILED,
                 context=catch(
                     context, status=FAILED, updated={"errors": to_dict(e)}
                 ),
-                info={"execution_time": time.monotonic() - ts},
-                extras=self.extras,
             )
         finally:
+            context["info"].update(
+                {
+                    "exec_end": get_dt_now(),
+                    "exec_latency": time.monotonic() - ts,
+                }
+            )
             trace.debug("[STAGE]: End Handler stage process.")
 
     async def _axecute(
@@ -905,6 +912,7 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
     `StageRetryError`.
     """
 
+    action_stage: ClassVar[bool] = True
     retry: int = Field(
         default=0,
         ge=0,
@@ -918,9 +926,8 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
     def _execute(
         self,
         params: DictData,
-        run_id: str,
         context: DictData,
-        parent_run_id: Optional[str] = None,
+        trace: Trace,
         event: Optional[Event] = None,
     ) -> Result:
         """Wrapped the execute method with retry strategy before returning to
@@ -938,9 +945,6 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
         current_retry: int = 0
         exception: Exception
         catch(context, status=WAIT)
-        trace: Trace = get_trace(
-            run_id, parent_run_id=parent_run_id, extras=self.extras
-        )
         # NOTE: First execution for not pass to retry step if it passes.
         try:
             if (
@@ -949,16 +953,16 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
             ):
                 return self.dryrun(
                     params | {"retry": current_retry},
-                    run_id=run_id,
+                    run_id=trace.run_id,
                     context=context,
-                    parent_run_id=parent_run_id,
+                    parent_run_id=trace.parent_run_id,
                     event=event,
                 )
             return self.process(
                 params | {"retry": current_retry},
-                run_id=run_id,
+                run_id=trace.run_id,
                 context=context,
-                parent_run_id=parent_run_id,
+                parent_run_id=trace.parent_run_id,
                 event=event,
             )
         except (
@@ -970,13 +974,11 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
             trace.debug("[STAGE]: process raise skip or cancel error.")
             raise
         except Exception as e:
+            if self.retry == 0:
+                raise
+
             current_retry += 1
             exception = e
-        finally:
-            trace.debug("[STAGE]: Failed at the first execution.")
-
-        if self.retry == 0:
-            raise exception
 
         trace.warning(
             f"[STAGE]: Retry count: {current_retry} ... "
@@ -996,16 +998,16 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
                 ):
                     return self.dryrun(
                         params | {"retry": current_retry},
-                        run_id=run_id,
+                        run_id=trace.run_id,
                         context=context,
-                        parent_run_id=parent_run_id,
+                        parent_run_id=trace.parent_run_id,
                         event=event,
                     )
                 return self.process(
                     params | {"retry": current_retry},
-                    run_id=run_id,
+                    run_id=trace.run_id,
                     context=context,
-                    parent_run_id=parent_run_id,
+                    parent_run_id=trace.parent_run_id,
                     event=event,
                 )
             except (
@@ -1086,13 +1088,11 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
             await trace.adebug("[STAGE]: process raise skip or cancel error.")
             raise
         except Exception as e:
+            if self.retry == 0:
+                raise
+
             current_retry += 1
             exception = e
-        finally:
-            await trace.adebug("[STAGE]: Failed at the first execution.")
-
-        if self.retry == 0:
-            raise exception
 
         await trace.awarning(
             f"[STAGE]: Retry count: {current_retry} ... "
@@ -1172,6 +1172,7 @@ class EmptyStage(BaseAsyncStage):
         ... })
     """
 
+    action_stage: ClassVar[bool] = True
     echo: StrOrNone = Field(
         default=None,
         description=(
@@ -1205,13 +1206,14 @@ class EmptyStage(BaseAsyncStage):
         without calling logging function.
 
         Args:
-            params: A parameter data that want to use in this
+            params (DictData): A parameter data that want to use in this
                 execution.
-            run_id: A running stage ID.
-            context: A context data.
-            parent_run_id: A parent running ID. (Default is None)
-            event: An event manager that use to track parent process
-                was not force stopped.
+            run_id (str): A running stage ID.
+            context (DictData): A context data that was passed from handler
+                method.
+            parent_run_id (str, default None): A parent running ID.
+            event (Event, default None): An event manager that use to track
+                parent process was not force stopped.
 
         Raises:
             StageCancelError: If event was set before start process.
@@ -1223,9 +1225,7 @@ class EmptyStage(BaseAsyncStage):
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         message: str = (
-            param2template(
-                dedent(self.echo.strip("\n")), params, extras=self.extras
-            )
+            self.pass_template(dedent(self.echo.strip("\n")), params=params)
             if self.echo
             else "..."
         )
@@ -1238,12 +1238,8 @@ class EmptyStage(BaseAsyncStage):
             if self.sleep > 5:
                 trace.info(f"[STAGE]: Sleep ... ({self.sleep} sec)")
             time.sleep(self.sleep)
-        return Result(
-            run_id=run_id,
-            parent_run_id=parent_run_id,
-            status=SUCCESS,
-            context=catch(context=context, status=SUCCESS),
-            extras=self.extras,
+        return Result.from_trace(trace).catch(
+            status=SUCCESS, context=catch(context=context, status=SUCCESS)
         )
 
     async def async_process(
@@ -1259,13 +1255,14 @@ class EmptyStage(BaseAsyncStage):
         stdout.
 
         Args:
-            params: A parameter data that want to use in this
+            params (DictData): A parameter data that want to use in this
                 execution.
-            run_id: A running stage ID.
-            context: A context data.
-            parent_run_id: A parent running ID. (Default is None)
-            event: An event manager that use to track parent process
-                was not force stopped.
+            run_id (str): A running stage ID.
+            context (DictData): A context data that was passed from handler
+                method.
+            parent_run_id (str, default None): A parent running ID.
+            event (Event, default None): An event manager that use to track
+                parent process was not force stopped.
 
         Raises:
             StageCancelError: If event was set before start process.
@@ -1277,9 +1274,7 @@ class EmptyStage(BaseAsyncStage):
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         message: str = (
-            param2template(
-                dedent(self.echo.strip("\n")), params, extras=self.extras
-            )
+            self.pass_template(dedent(self.echo.strip("\n")), params=params)
             if self.echo
             else "..."
         )
@@ -1292,12 +1287,8 @@ class EmptyStage(BaseAsyncStage):
             if self.sleep > 5:
                 await trace.ainfo(f"[STAGE]: Sleep ... ({self.sleep} sec)")
             await asyncio.sleep(self.sleep)
-        return Result(
-            run_id=run_id,
-            parent_run_id=parent_run_id,
-            status=SUCCESS,
-            context=catch(context=context, status=SUCCESS),
-            extras=self.extras,
+        return Result.from_trace(trace).catch(
+            status=SUCCESS, context=catch(context=context, status=SUCCESS)
         )
 
 
@@ -1322,7 +1313,6 @@ class BashStage(BaseRetryStage):
         ... })
     """
 
-    action_stage: ClassVar[bool] = True
     bash: str = Field(
         description=(
             "A bash statement that want to execute via Python subprocess."
@@ -1370,10 +1360,11 @@ class BashStage(BaseRetryStage):
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
 
-        yield f_shebang, f_name
-
-        # Note: Remove .sh file that use to run bash.
-        Path(f"./{f_name}").unlink()
+        try:
+            yield f_shebang, f_name
+        finally:
+            # Note: Remove .sh file that use to run bash.
+            Path(f"./{f_name}").unlink()
 
     @contextlib.contextmanager
     def make_sh_file(
@@ -1408,10 +1399,11 @@ class BashStage(BaseRetryStage):
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
 
-        yield f_shebang, f_name
-
-        # Note: Remove .sh file that use to run bash.
-        Path(f"./{f_name}").unlink()
+        try:
+            yield f_shebang, f_name
+        finally:
+            # Note: Remove .sh file that use to run bash.
+            Path(f"./{f_name}").unlink()
 
     @staticmethod
     def prepare_std(value: str) -> Optional[str]:
@@ -1592,7 +1584,6 @@ class PyStage(BaseRetryStage):
         ... })
     """
 
-    action_stage: ClassVar[bool] = True
     run: str = Field(
         description="A Python string statement that want to run with `exec`.",
     )
@@ -1622,6 +1613,7 @@ class PyStage(BaseRetryStage):
                 or (value.startswith("__") and value.endswith("__"))
                 or ismodule(values[value])
                 or isclass(values[value])
+                or value in ("trace",)
             ):
                 continue
 
@@ -1831,7 +1823,6 @@ class CallStage(BaseRetryStage):
         ... })
     """
 
-    action_stage: ClassVar[bool] = True
     uses: str = Field(
         description=(
             "A caller function with registry importer syntax that use to load "
@@ -2337,10 +2328,10 @@ class TriggerStage(BaseRetryStage):
                 if (msg := result.context.get("errors", {}).get("message"))
                 else "."
             )
-            err = StageError(
-                f"Trigger workflow was failed{err_msg}", allow_traceback=False
+            raise StageError(
+                f"Trigger workflow was failed{err_msg}",
+                allow_traceback=False,
             )
-            raise err
         elif result.status == CANCEL:
             raise StageCancelError("Trigger workflow was cancel.")
         elif result.status == SKIP:
@@ -2405,8 +2396,8 @@ class BaseNestedStage(BaseAsyncStage, ABC):
         execute func.
 
         Args:
-            context: (DictData) A context data.
-            error: (StageError) A stage exception object.
+            context (DictData): A context data.
+            error (StageError): A stage exception object.
         """
         if "errors" in context:
             context["errors"][error.refs] = error.to_dict()
@@ -2425,13 +2416,14 @@ class BaseNestedStage(BaseAsyncStage, ABC):
         """Async process for nested-stage do not implement yet.
 
         Args:
-            params: A parameter data that want to use in this
+            params (DictData): A parameter data that want to use in this
                 execution.
-            run_id: A running stage ID.
-            context: A context data.
-            parent_run_id: A parent running ID. (Default is None)
-            event: An event manager that use to track parent process
-                was not force stopped.
+            run_id (str): A running stage ID.
+            context (DictData): A context data that was passed from handler
+                method.
+            parent_run_id (str, default None): A parent running ID.
+            event (Event, default None): An event manager that use to track
+                parent process was not force stopped.
 
         Returns:
             Result: The execution result with status and context data.