ddeutil-workflow 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -45,7 +45,7 @@ from .__types import DictData, DictStr, TupleStr
 from .conf import config, get_logger
 from .exceptions import StageException
 from .hook import TagFunc, extract_hook
-from .result import Result
+from .result import Result, Status
 from .templates import not_in_template, param2template
 from .utils import (
     cut_id,
@@ -121,19 +121,26 @@ class BaseStage(BaseModel, ABC):
         return self
 
     @abstractmethod
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
     def handler_execute(
-        self, params: DictData, *, run_id: str | None = None
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        result: Result | None = None,
     ) -> Result:
         """Handler result from the stage execution.
 
@@ -158,23 +165,25 @@ class BaseStage(BaseModel, ABC):
         from current stage ID before release the final result.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param run_id: (str) A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        if not run_id:
-            run_id: str = gen_id(self.name + (self.id or ""), unique=True)
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=(
+                    run_id or gen_id(self.name + (self.id or ""), unique=True)
+                ),
+            )
 
-        rs_raise: Result = Result(status=1, run_id=run_id)
         try:
             # NOTE: Start calling origin function with a passing args.
-            return self.execute(params, run_id=run_id)
+            return self.execute(params, result=result)
         except Exception as err:
             # NOTE: Start catching error from the stage execution.
-            logger.error(
-                f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
-                f"{err}"
-            )
+            result.trace.error(f"[STAGE]: {err.__class__.__name__}: {err}")
             if config.stage_raise_error:
                 # NOTE: If error that raise from stage execution course by
                 #   itself, it will return that error with previous
@@ -190,8 +199,8 @@ class BaseStage(BaseModel, ABC):
 
             # NOTE: Catching exception error object to result with
             #   error_message and error keys.
-            return rs_raise.catch(
-                status=1,
+            return result.catch(
+                status=Status.FAILED,
                 context={
                     "error": err,
                     "error_message": f"{err.__class__.__name__}: {err}",
@@ -295,7 +304,9 @@ class EmptyStage(BaseStage):
         ge=0,
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
         it does not get any error from logging function.
@@ -305,22 +316,21 @@ class EmptyStage(BaseStage):
 
         :param params: A context data that want to add output result. But this
             stage does not pass any output.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
+        result.trace.info(
+            f"[STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
             if self.sleep > 30:
-                logger.info(
-                    f"({cut_id(run_id)}) [STAGE]: ... sleep "
-                    f"({self.sleep} seconds)"
-                )
+                result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
             time.sleep(self.sleep)
-        return Result(status=0, context={}, run_id=run_id)
+
+        return result.catch(status=Status.SUCCESS)
 
 
 class BashStage(BaseStage):
@@ -334,7 +344,7 @@ class BashStage(BaseStage):
 
     Data Validate:
         >>> stage = {
-        ...     "name": "Shell stage execution",
+        ...     "name": "The Shell stage execution",
         ...     "bash": 'echo "Hello $FOO"',
         ...     "env": {
         ...         "FOO": "BAR",
@@ -391,20 +401,25 @@ class BashStage(BaseStage):
         # Note: Remove .sh file that use to run bash.
         Path(f"./{f_name}").unlink()
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
+        result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
-            bash=bash, env=param2template(self.env, params), run_id=run_id
+            bash=bash,
+            env=param2template(self.env, params),
+            run_id=result.run_id,
         ) as sh:
             rs: CompletedProcess = subprocess.run(
                 sh, shell=False, capture_output=True, text=True
@@ -420,14 +435,13 @@ class BashStage(BaseStage):
                 f"Subprocess: {err}\nRunning Statement:\n---\n"
                 f"```bash\n{bash}\n```"
             )
-        return Result(
-            status=0,
+        return result.catch(
+            status=Status.SUCCESS,
             context={
                 "return_code": rs.returncode,
                 "stdout": rs.stdout.rstrip("\n") or None,
                 "stderr": rs.stderr.rstrip("\n") or None,
             },
-            run_id=run_id,
         )
 
 
@@ -492,12 +506,15 @@ class PyStage(BaseStage):
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
 
         :param params: A parameter that want to pass before run any statement.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
@@ -511,16 +528,14 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
+        result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
         exec(run, _globals, lc)
 
-        return Result(
-            status=0,
-            context={"locals": lc, "globals": _globals},
-            run_id=run_id,
+        return result.catch(
+            status=Status.SUCCESS, context={"locals": lc, "globals": _globals}
         )
 
 
@@ -552,7 +567,9 @@ class HookStage(BaseStage):
         alias="with",
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Hook function that already in the hook registry.
 
         :raise ValueError: When the necessary arguments of hook function do not
@@ -562,7 +579,8 @@ class HookStage(BaseStage):
 
         :param params: A parameter that want to pass before run any statement.
         :type params: DictData
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :type: str | None
 
         :rtype: Result
@@ -571,7 +589,7 @@ class HookStage(BaseStage):
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = param2template(self.args, params)
+        args: DictData = {"result": result} | param2template(self.args, params)
         ips = inspect.signature(t_func)
         if any(
             (k.removeprefix("_") not in args and k not in args)
@@ -587,10 +605,10 @@ class HookStage(BaseStage):
             if k.removeprefix("_") in args:
                 args[k] = args.pop(k.removeprefix("_"))
 
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
-            f"{t_func.name}@{t_func.tag}"
-        )
+        if "result" not in ips.parameters:
+            args.pop("result")
+
+        result.trace.info(f"[STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}")
         rs: DictData = t_func(**param2template(args, params))
 
         # VALIDATE:
@@ -600,7 +618,7 @@ class HookStage(BaseStage):
                 f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
                 f"to result model, you change return type to `dict`."
             )
-        return Result(status=0, context=rs, run_id=run_id)
+        return result.catch(status=Status.SUCCESS, context=rs)
 
 
 class TriggerStage(BaseStage):
@@ -626,12 +644,15 @@ class TriggerStage(BaseStage):
         description="A parameter that want to pass to workflow execution.",
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
@@ -644,13 +665,11 @@ class TriggerStage(BaseStage):
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
         wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
-        )
+        result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
         return wf.execute(
             params=param2template(self.params, params),
-            run_id=run_id,
-        ).set_run_id(run_id)
+            result=result,
+        )
 
 
 # NOTE:

ddeutil/workflow/workflow.py

@@ -43,7 +43,8 @@ from typing_extensions import Self
 
 from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, Log, config, get_log, get_logger
+from .audit import Audit, get_audit
+from .conf import Loader, config, get_logger
 from .cron import On
 from .exceptions import JobException, WorkflowException
 from .job import Job
@@ -485,7 +486,7 @@ class Workflow(BaseModel):
         params: DictData,
         *,
         run_id: str | None = None,
-        log: type[Log] = None,
+        log: type[Audit] = None,
         queue: ReleaseQueue | None = None,
         override_log_name: str | None = None,
     ) -> Result:
@@ -515,7 +516,7 @@ class Workflow(BaseModel):
 
         :rtype: Result
         """
-        log: type[Log] = log or get_log()
+        log: type[Audit] = log or get_audit()
         name: str = override_log_name or self.name
         run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
@@ -562,15 +563,14 @@ class Workflow(BaseModel):
         # NOTE: Saving execution result to destination of the input log object.
         logger.debug(f"({cut_id(run_id)}) [LOG]: Writing log: {name!r}.")
         (
-            log.model_validate(
-                {
-                    "name": name,
-                    "release": release.date,
-                    "type": release.type,
-                    "context": rs.context,
-                    "parent_run_id": rs.parent_run_id,
-                    "run_id": rs.run_id,
-                }
+            log(
+                name=name,
+                release=release.date,
+                type=release.type,
+                context=rs.context,
+                parent_run_id=rs.parent_run_id,
+                run_id=rs.run_id,
+                execution_time=rs.alive_time(),
             ).save(excluded=None)
         )
 
@@ -602,7 +602,7 @@ class Workflow(BaseModel):
         offset: float,
         end_date: datetime,
         queue: ReleaseQueue,
-        log: type[Log],
+        log: type[Audit],
         *,
         force_run: bool = False,
     ) -> ReleaseQueue:
@@ -671,7 +671,7 @@ class Workflow(BaseModel):
         *,
         run_id: str | None = None,
         periods: int = 1,
-        log: Log | None = None,
+        log: Audit | None = None,
         force_run: bool = False,
         timeout: int = 1800,
     ) -> list[Result]:
@@ -698,7 +698,7 @@ class Workflow(BaseModel):
         :rtype: list[Result]
         :return: A list of all results that return from ``self.release`` method.
         """
-        log: type[Log] = log or get_log()
+        log: type[Audit] = log or get_audit()
         run_id: str = run_id or gen_id(self.name, unique=True)
 
         # VALIDATE: Check the periods value should gather than 0.
@@ -820,6 +820,7 @@ class Workflow(BaseModel):
         *,
         run_id: str | None = None,
         raise_error: bool = True,
+        result: Result | None = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the main workflow
         execution to the target job object via job's ID.
@@ -837,13 +838,18 @@ class Workflow(BaseModel):
         :param run_id: A workflow running ID for this job execution.
         :param raise_error: A flag that raise error instead catching to result
             if it gets exception from job execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         :return: Return the result object that receive the job execution result
             context.
         """
-        run_id: str = run_id or gen_id(self.name, unique=True)
-        rs: Result = Result(run_id=run_id)
+        if result is None:  # pragma: no cov
+            run_id: str = run_id or gen_id(self.name, unique=True)
+            result: Result = Result(run_id=run_id)
+        else:
+            run_id: str = result.run_id
 
         # VALIDATE: check a job ID that exists in this workflow or not.
         if job_id not in self.jobs:
@@ -852,9 +858,7 @@ class Workflow(BaseModel):
                 f"workflow."
             )
 
-        logger.info(
-            f"({cut_id(run_id)}) [WORKFLOW]: Start execute job: {job_id!r}"
-        )
+        result.trace.info(f"[WORKFLOW]: Start execute job: {job_id!r}")
 
         # IMPORTANT:
         #   This execution change all job running IDs to the current workflow
@@ -868,10 +872,7 @@ class Workflow(BaseModel):
                 to=params,
             )
         except JobException as err:
-            logger.error(
-                f"({cut_id(run_id)}) [WORKFLOW]: {err.__class__.__name__}: "
-                f"{err}"
-            )
+            result.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
             if raise_error:
                 raise WorkflowException(
                     f"Get job execution error {job_id}: JobException: {err}"
@@ -880,7 +881,7 @@ class Workflow(BaseModel):
                 "Handle error from the job execution does not support yet."
             ) from None
 
-        return rs.catch(status=0, context=params)
+        return result.catch(status=0, context=params)
 
     def execute(
         self,
@@ -888,6 +889,7 @@ class Workflow(BaseModel):
         *,
         run_id: str | None = None,
         timeout: int = 0,
+        result: Result | None = None,
     ) -> Result:
         """Execute workflow with passing a dynamic parameters to all jobs that
         included in this workflow model with ``jobs`` field.
@@ -907,31 +909,32 @@ class Workflow(BaseModel):
 
         :param run_id: A workflow running ID for this job execution.
         :type run_id: str | None (default: None)
-        :param timeout: A workflow execution time out in second unit that use
+        :param timeout: (int) A workflow execution time out in second unit that use
             for limit time of execution and waiting job dependency. This value
             does not force stop the task that still running more than this limit
-            time.
-        :type timeout: int (default: 0)
+            time. (default: 0)
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        run_id: str = run_id or gen_id(self.name, unique=True)
-        logger.info(
-            f"({cut_id(run_id)}) [WORKFLOW]: Start Execute: {self.name!r} ..."
-        )
-
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
         ts: float = time.monotonic()
-        rs: Result = Result(run_id=run_id)
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=(run_id or gen_id(self.name, unique=True))
+            )
+
+        result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
 
         # NOTE: It should not do anything if it does not have job.
         if not self.jobs:
-            logger.warning(
-                f"({cut_id(run_id)}) [WORKFLOW]: This workflow: {self.name!r} "
-                f"does not have any jobs"
+            result.trace.warning(
+                f"[WORKFLOW]: This workflow: {self.name!r} does not have any "
+                f"jobs"
             )
-            return rs.catch(status=0, context=params)
+            return result.catch(status=0, context=params)
 
         # NOTE: Create a job queue that keep the job that want to run after
         #   its dependency condition.
@@ -952,7 +955,7 @@ class Workflow(BaseModel):
         try:
             if config.max_job_parallel == 1:
                 self.__exec_non_threading(
-                    run_id=run_id,
+                    result=result,
                     context=context,
                     ts=ts,
                     job_queue=jq,
@@ -960,7 +963,7 @@ class Workflow(BaseModel):
                 )
             else:
                 self.__exec_threading(
-                    run_id=run_id,
+                    result=result,
                     context=context,
                     ts=ts,
                     job_queue=jq,
@@ -974,11 +977,11 @@ class Workflow(BaseModel):
                     "error_message": f"{err.__class__.__name__}: {err}",
                 },
             )
-        return rs.catch(status=status, context=context)
+        return result.catch(status=status, context=context)
 
     def __exec_threading(
         self,
-        run_id: str,
+        result: Result,
         context: DictData,
         ts: float,
         job_queue: Queue,
@@ -991,6 +994,7 @@ class Workflow(BaseModel):
             If a job need dependency, it will check dependency job ID from
         context data before allow it run.
 
+        :param result: A result model.
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
             time out.
@@ -1002,9 +1006,7 @@ class Workflow(BaseModel):
         """
         not_timeout_flag: bool = True
         timeout: int = timeout or config.max_job_exec_timeout
-        logger.debug(
-            f"({cut_id(run_id)}) [WORKFLOW]: Run {self.name!r} with threading."
-        )
+        result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
 
         # IMPORTANT: The job execution can run parallel and waiting by
         #   needed.
@@ -1041,6 +1043,7 @@ class Workflow(BaseModel):
                         self.execute_job,
                         job_id,
                         params=context,
+                        result=result,
                     ),
                 )
 
@@ -1055,7 +1058,7 @@ class Workflow(BaseModel):
 
                 for future in as_completed(futures, timeout=thread_timeout):
                     if err := future.exception():
-                        logger.error(f"({cut_id(run_id)}) [WORKFLOW]: {err}")
+                        result.trace.error(f"[WORKFLOW]: {err}")
                         raise WorkflowException(str(err))
 
                     # NOTE: This getting result does not do anything.
@@ -1067,15 +1070,14 @@ class Workflow(BaseModel):
                 future.cancel()
 
         # NOTE: Raise timeout error.
-        logger.warning(
-            f"({cut_id(run_id)}) [WORKFLOW]: Execution: {self.name!r} "
-            f"was timeout."
+        result.trace.warning(
+            f"[WORKFLOW]: Execution: {self.name!r} was timeout."
         )
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
     def __exec_non_threading(
         self,
-        run_id: str,
+        result: Result,
         context: DictData,
         ts: float,
         job_queue: Queue,
@@ -1088,6 +1090,7 @@ class Workflow(BaseModel):
             If a job need dependency, it will check dependency job ID from
         context data before allow it run.
 
+        :param result: A result model.
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
             time out.
@@ -1097,10 +1100,7 @@ class Workflow(BaseModel):
         """
         not_timeout_flag: bool = True
         timeout: int = timeout or config.max_job_exec_timeout
-        logger.debug(
-            f"({cut_id(run_id)}) [WORKFLOW]: Run {self.name!r} with "
-            f"non-threading."
-        )
+        result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
 
         while not job_queue.empty() and (
             not_timeout_flag := ((time.monotonic() - ts) < timeout)
@@ -1123,7 +1123,7 @@ class Workflow(BaseModel):
             #       'params': <input-params>,
             #       'jobs': {},
             #   }
-            self.execute_job(job_id=job_id, params=context, run_id=run_id)
+            self.execute_job(job_id=job_id, params=context, result=result)
 
             # NOTE: Mark this job queue done.
             job_queue.task_done()
@@ -1137,9 +1137,8 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
-            f"({cut_id(run_id)}) [WORKFLOW]: Execution: {self.name!r} "
-            f"was timeout."
+        result.trace.warning(
+            f"[WORKFLOW]: Execution: {self.name!r} was timeout."
         )
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
@@ -1165,7 +1164,7 @@ class WorkflowTask:
         self,
         release: datetime | Release | None = None,
         run_id: str | None = None,
-        log: type[Log] = None,
+        log: type[Audit] = None,
         queue: ReleaseQueue | None = None,
     ) -> Result:
         """Release the workflow task data.
@@ -1177,7 +1176,7 @@ class WorkflowTask:
 
         :rtype: Result
         """
-        log: type[Log] = log or get_log()
+        log: type[Audit] = log or get_audit()
 
         if release is None:
 
@@ -1213,7 +1212,7 @@ class WorkflowTask:
         self,
         end_date: datetime,
         queue: ReleaseQueue,
-        log: type[Log],
+        log: type[Audit],
         *,
         force_run: bool = False,
     ) -> ReleaseQueue:
@@ -1223,8 +1222,8 @@ class WorkflowTask:
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
         :param log: A log class that want to make log object.
-        :param force_run: A flag that allow to release workflow if the log with
-            that release was pointed.
+        :param force_run: (bool) A flag that allow to release workflow if the
+            log with that release was pointed.
 
         :rtype: ReleaseQueue
         """
@@ -1260,7 +1259,7 @@ class WorkflowTask:
         return queue
 
     def __repr__(self) -> str:
-        """Override ___repr__ method."""
+        """Override the `__repr__` method."""
         return (
             f"{self.__class__.__name__}(alias={self.alias!r}, "
             f"workflow={self.workflow.name!r}, runner={self.runner!r}, "