ddeutil-workflow 0.0.19__py3-none-any.whl → 0.0.21__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -13,8 +13,9 @@ handle stage error on this stage model. I think stage model should have a lot of
 usecase and it does not worry when I want to create a new one.
 
     Execution   --> Ok      --> Result with 0
+
                 --> Error   --> Result with 1 (if env var was set)
-                            --> Raise StageException
+                            --> Raise StageException(...)
 
     On the context I/O that pass to a stage object at execute process. The
 execute method receives a `params={"params": {...}}` value for mapping to
@@ -68,16 +69,13 @@ logger = get_logger("ddeutil.workflow")
 
 
 __all__: TupleStr = (
-    "BaseStage",
     "EmptyStage",
     "BashStage",
     "PyStage",
     "HookStage",
     "TriggerStage",
     "Stage",
-    "HookSearchData",
     "extract_hook",
-    "handler_result",
 )
 
 
@@ -90,15 +88,17 @@ def handler_result(message: str | None = None) -> DecoratorResult:
     environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
 
         Execution   --> Ok      --> Result
-                                        status: 0
-                                        context:
-                                            outputs: ...
+                                    |-status: 0
+                                    |-context:
+                                        |-outputs: ...
+
                     --> Error   --> Result (if env var was set)
-                                        status: 1
-                                        context:
-                                            error: ...
-                                            error_message: ...
-                    --> Error   --> Raise StageException
+                                    |-status: 1
+                                    |-context:
+                                        |-error: ...
+                                        |-error_message: ...
+
+                    --> Error   --> Raise StageException(...)
 
         On the last step, it will set the running ID on a return result object
     from current stage ID before release the final result.
@@ -119,13 +119,18 @@ def handler_result(message: str | None = None) -> DecoratorResult:
 
         @wraps(func)
         def wrapped(self: Stage, *args, **kwargs):
+
+            if not (run_id := kwargs.get("run_id")):
+                run_id: str = gen_id(self.name + (self.id or ""), unique=True)
+                kwargs["run_id"] = run_id
+
             try:
                 # NOTE: Start calling origin function with a passing args.
-                return func(self, *args, **kwargs).set_run_id(self.run_id)
+                return func(self, *args, **kwargs)
             except Exception as err:
                 # NOTE: Start catching error from the stage execution.
                 logger.error(
-                    f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
+                    f"({run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
@@ -148,7 +153,8 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                         "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                ).set_run_id(self.run_id)
+                    run_id=run_id,
+                )
 
         return wrapped
 
@@ -159,6 +165,8 @@ class BaseStage(BaseModel, ABC):
     """Base Stage Model that keep only id and name fields for the stage
     metadata. If you want to implement any custom stage, you can use this class
     to parent and implement ``self.execute()`` method only.
+
+        This class is the abstraction class for any stage class.
     """
 
     id: Optional[str] = Field(
@@ -176,12 +184,15 @@ class BaseStage(BaseModel, ABC):
         description="A stage condition statement to allow stage executable.",
         alias="if",
     )
-    run_id: Optional[str] = Field(
-        default=None,
-        description="A running stage ID.",
-        repr=False,
-        exclude=True,
-    )
+
+    @property
+    def iden(self) -> str:
+        """Return identity of this stage object that return the id field first.
+        If the id does not set, it will use name field instead.
+
+        :rtype: str
+        """
+        return self.id or self.name
 
     @model_validator(mode="after")
     def __prepare_running_id__(self) -> Self:
@@ -194,8 +205,6 @@ class BaseStage(BaseModel, ABC):
 
         :rtype: Self
         """
-        if self.run_id is None:
-            self.run_id = gen_id(self.name + (self.id or ""), unique=True)
 
         # VALIDATE: Validate stage id and name should not dynamic with params
         #   template. (allow only matrix)
@@ -206,21 +215,14 @@ class BaseStage(BaseModel, ABC):
 
         return self
 
-    def get_running_id(self, run_id: str) -> Self:
-        """Return Stage model object that changing stage running ID with an
-        input running ID.
-
-        :param run_id: A replace stage running ID.
-        :rtype: Self
-        """
-        return self.model_copy(update={"run_id": run_id})
-
     @abstractmethod
-    def execute(self, params: DictData) -> Result:
+    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
 
         :param params: A parameter data that want to use in this execution.
+        :param run_id: A running stage ID for this execution.
+
         :rtype: Result
         """
         raise NotImplementedError("Stage should implement ``execute`` method.")
@@ -248,10 +250,9 @@ class BaseStage(BaseModel, ABC):
         :rtype: DictData
         """
         if self.id is None and not config.stage_default_id:
-            logger.debug(
-                f"({self.run_id}) [STAGE]: Output does not set because this "
-                f"stage does not set ID or default stage ID config flag not be "
-                f"True."
+            logger.warning(
+                "Output does not set because this stage does not set ID or "
+                "default stage ID config flag not be True."
             )
             return to
 
@@ -267,7 +268,6 @@ class BaseStage(BaseModel, ABC):
         )
 
         # NOTE: Set the output to that stage generated ID with ``outputs`` key.
-        logger.debug(f"({self.run_id}) [STAGE]: Set outputs to {_id!r}")
         to["stages"][_id] = {"outputs": output}
         return to
 
@@ -283,18 +283,22 @@ class BaseStage(BaseModel, ABC):
         :param params: A parameters that want to pass to condition template.
         :rtype: bool
         """
+        # NOTE: Return false result if condition does not set.
         if self.condition is None:
             return False
 
         params: DictData = {} if params is None else params
-        _g: DictData = globals() | params
+
         try:
-            rs: bool = eval(param2template(self.condition, params), _g, {})
+            # WARNING: The eval build-in function is vary dangerous. So, it
+            #   should us the re module to validate eval-string before running.
+            rs: bool = eval(
+                param2template(self.condition, params), globals() | params, {}
+            )
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
             return not rs
         except Exception as err:
-            logger.error(f"({self.run_id}) [STAGE]: {err}")
             raise StageException(f"{err.__class__.__name__}: {err}") from err
 
 
@@ -319,7 +323,8 @@ class EmptyStage(BaseStage):
         ge=0,
     )
 
-    def execute(self, params: DictData) -> Result:
+    @handler_result()
+    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
         it does not get any error from logging function.
@@ -329,15 +334,17 @@ class EmptyStage(BaseStage):
 
         :param params: A context data that want to add output result. But this
             stage does not pass any output.
+        :param run_id: A running stage ID for this execution.
+
         :rtype: Result
         """
         logger.info(
-            f"({self.run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
+            f"({run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
             time.sleep(self.sleep)
-        return Result(status=0, context={})
+        return Result(status=0, context={}, run_id=run_id)
 
 
 class BashStage(BaseStage):
@@ -369,17 +376,25 @@ class BashStage(BaseStage):
     )
 
     @contextlib.contextmanager
-    def prepare_bash(self, bash: str, env: DictStr) -> Iterator[TupleStr]:
+    def create_sh_file(
+        self, bash: str, env: DictStr, run_id: str | None = None
+    ) -> Iterator[TupleStr]:
         """Return context of prepared bash statement that want to execute. This
         step will write the `.sh` file before giving this file name to context.
         After that, it will auto delete this file automatic.
 
         :param bash: A bash statement that want to execute.
         :param env: An environment variable that use on this bash statement.
+        :param run_id: A running stage ID that use for writing sh file instead
+            generate by UUID4.
         :rtype: Iterator[TupleStr]
         """
-        f_name: str = f"{uuid.uuid4()}.sh"
+        run_id: str = run_id or uuid.uuid4()
+        f_name: str = f"{run_id}.sh"
         f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
+
+        logger.debug(f"({run_id}) [STAGE]: Start create `{f_name}` file.")
+
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: write header of `.sh` file
             f.write(f"#!/bin/{f_shebang}\n\n")
@@ -393,29 +408,27 @@ class BashStage(BaseStage):
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
 
-        logger.debug(
-            f"({self.run_id}) [STAGE]: Start create `.sh` file and running a "
-            f"bash statement."
-        )
-
         yield [f_shebang, f_name]
 
         # Note: Remove .sh file that use to run bash.
         Path(f"./{f_name}").unlink()
 
     @handler_result()
-    def execute(self, params: DictData) -> Result:
+    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
 
         :param params: A parameter data that want to use in this execution.
+        :param run_id: A running stage ID for this execution.
+
         :rtype: Result
         """
         bash: str = param2template(dedent(self.bash), params)
-        with self.prepare_bash(
-            bash=bash, env=param2template(self.env, params)
+
+        logger.info(f"({run_id}) [STAGE]: Shell-Execute: {self.name}")
+        with self.create_sh_file(
+            bash=bash, env=param2template(self.env, params), run_id=run_id
         ) as sh:
-            logger.info(f"({self.run_id}) [STAGE]: Shell-Execute: {sh}")
             rs: CompletedProcess = subprocess.run(
                 sh, shell=False, capture_output=True, text=True
             )
@@ -437,6 +450,7 @@ class BashStage(BaseStage):
                 "stdout": rs.stdout.rstrip("\n") or None,
                 "stderr": rs.stderr.rstrip("\n") or None,
             },
+            run_id=run_id,
         )
 
 
@@ -482,6 +496,7 @@ class PyStage(BaseStage):
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         # NOTE: The output will fileter unnecessary keys from locals.
@@ -501,11 +516,13 @@ class PyStage(BaseStage):
         return to
 
     @handler_result()
-    def execute(self, params: DictData) -> Result:
+    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
 
         :param params: A parameter that want to pass before run any statement.
+        :param run_id: A running stage ID for this execution.
+
         :rtype: Result
         """
         # NOTE: Replace the run statement that has templating value.
@@ -518,12 +535,16 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({self.run_id}) [STAGE]: Py-Execute: {self.name}")
+        logger.info(f"({run_id}) [STAGE]: Py-Execute: {self.name}")
+
+        # WARNING: The exec build-in function is vary dangerous. So, it
+        #   should us the re module to validate exec-string before running.
         exec(run, _globals, lc)
 
         return Result(
             status=0,
             context={"locals": lc, "globals": _globals},
+            run_id=run_id,
         )
 
 
@@ -603,7 +624,7 @@ class HookStage(BaseStage):
     )
 
     @handler_result()
-    def execute(self, params: DictData) -> Result:
+    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Hook function that already in the hook registry.
 
         :raise ValueError: When the necessary arguments of hook function do not
@@ -613,10 +634,12 @@ class HookStage(BaseStage):
 
         :param params: A parameter that want to pass before run any statement.
         :type params: DictData
+        :param run_id: A running stage ID for this execution.
+        :type: str | None
+
         :rtype: Result
         """
-        t_func_hook: str = param2template(self.uses, params)
-        t_func: TagFunc = extract_hook(t_func_hook)()
+        t_func: TagFunc = extract_hook(param2template(self.uses, params))()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
@@ -637,7 +660,7 @@ class HookStage(BaseStage):
                 args[k] = args.pop(k.removeprefix("_"))
 
         logger.info(
-            f"({self.run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
+            f"({run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
         )
         rs: DictData = t_func(**param2template(args, params))
 
@@ -648,7 +671,7 @@ class HookStage(BaseStage):
                 f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
                 f"to result model, you change return type to `dict`."
             )
-        return Result(status=0, context=rs)
+        return Result(status=0, context=rs, run_id=run_id)
 
 
 class TriggerStage(BaseStage):
@@ -675,11 +698,13 @@ class TriggerStage(BaseStage):
     )
 
     @handler_result("Raise from TriggerStage")
-    def execute(self, params: DictData) -> Result:
+    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Trigger another workflow execution. It will waiting the trigger
         workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
+        :param run_id: A running stage ID for this execution.
+
         :rtype: Result
         """
         # NOTE: Lazy import this workflow object.
@@ -690,11 +715,12 @@ class TriggerStage(BaseStage):
 
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
-        wf: Workflow = Workflow.from_loader(
-            name=_trigger, externals={"run_id": self.run_id}
-        )
-        logger.info(f"({self.run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
-        return wf.execute(params=param2template(self.params, params))
+        wf: Workflow = Workflow.from_loader(name=_trigger)
+        logger.info(f"({run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
+        return wf.execute(
+            params=param2template(self.params, params),
+            run_id=run_id,
+        ).set_run_id(run_id)
 
 
 # NOTE:

ddeutil/workflow/utils.py

@@ -13,7 +13,7 @@ from abc import ABC, abstractmethod
 from ast import Call, Constant, Expr, Module, Name, parse
 from collections.abc import Iterator
 from dataclasses import field
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 from functools import wraps
 from hashlib import md5
 from importlib import import_module
@@ -48,26 +48,34 @@ AnyModelType = type[AnyModel]
 logger = logging.getLogger("ddeutil.workflow")
 
 
-def get_dt_now(tz: ZoneInfo | None = None) -> datetime:  # pragma: no cov
+def get_dt_now(
+    tz: ZoneInfo | None = None, offset: float = 0.0
+) -> datetime:  # pragma: no cov
     """Return the current datetime object.
 
     :param tz:
+    :param offset:
     :return: The current datetime object that use an input timezone or UTC.
     """
-    return datetime.now(tz=(tz or ZoneInfo("UTC")))
+    return datetime.now(tz=(tz or ZoneInfo("UTC"))) - timedelta(seconds=offset)
 
 
 def get_diff_sec(
-    dt: datetime, tz: ZoneInfo | None = None
+    dt: datetime, tz: ZoneInfo | None = None, offset: float = 0.0
 ) -> int:  # pragma: no cov
     """Return second value that come from diff of an input datetime and the
     current datetime with specific timezone.
 
     :param dt:
     :param tz:
+    :param offset:
     """
     return round(
-        (dt - datetime.now(tz=(tz or ZoneInfo("UTC")))).total_seconds()
+        (
+            dt
+            - datetime.now(tz=(tz or ZoneInfo("UTC")))
+            - timedelta(seconds=offset)
+        ).total_seconds()
     )
 
 
@@ -350,12 +358,10 @@ class Result:
 
     status: int = field(default=2)
     context: DictData = field(default_factory=dict)
-    start_at: datetime = field(default_factory=get_dt_now, compare=False)
-    end_at: Optional[datetime] = field(default=None, compare=False)
 
     # NOTE: Ignore this field to compare another result model with __eq__.
-    _run_id: Optional[str] = field(default=None)
-    _parent_run_id: Optional[str] = field(default=None, compare=False)
+    run_id: Optional[str] = field(default=None)
+    parent_run_id: Optional[str] = field(default=None, compare=False)
 
     @model_validator(mode="after")
     def __prepare_run_id(self) -> Self:
@@ -373,7 +379,7 @@ class Result:
         :param running_id: A running ID that want to update on this model.
         :rtype: Self
         """
-        self._run_id = running_id
+        self.run_id = running_id
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
@@ -382,17 +388,9 @@ class Result:
         :param running_id: A running ID that want to update on this model.
         :rtype: Self
         """
-        self._parent_run_id: str = running_id
+        self.parent_run_id: str = running_id
         return self
 
-    @property
-    def parent_run_id(self) -> str:
-        return self._parent_run_id
-
-    @property
-    def run_id(self) -> str:
-        return self._run_id
-
     def catch(self, status: int, context: DictData) -> Self:
         """Catch the status and context to current data."""
         self.__dict__["status"] = status
@@ -408,8 +406,8 @@ class Result:
         self.__dict__["context"].update(result.context)
 
         # NOTE: Update running ID from an incoming result.
-        self._parent_run_id = result.parent_run_id
-        self._run_id = result.run_id
+        self.parent_run_id = result.parent_run_id
+        self.run_id = result.run_id
         return self
 
     def receive_jobs(self, result: Result) -> Self:
@@ -427,8 +425,8 @@ class Result:
         self.__dict__["context"]["jobs"].update(result.context)
 
         # NOTE: Update running ID from an incoming result.
-        self._parent_run_id: str = result.parent_run_id
-        self._run_id: str = result.run_id
+        self.parent_run_id: str = result.parent_run_id
+        self.run_id: str = result.run_id
         return self
 
 
@@ -576,7 +574,14 @@ def get_args_from_filter(
 
 @custom_filter("fmt")  # pragma: no cov
 def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
-    """Format datetime object to string with the format."""
+    """Format datetime object to string with the format.
+
+    :param value: A datetime value that want to format to string value.
+    :param fmt: A format string pattern that passing to the `dt.strftime`
+        method.
+
+    :rtype: str
+    """
     if isinstance(value, datetime):
         return value.strftime(fmt)
     raise UtilException(