ddeutil-workflow 0.0.33__py3-none-any.whl → 0.0.35__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -3,12 +3,12 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""The main schedule running is ``workflow_runner`` function that trigger the
-multiprocess of ``workflow_control`` function for listing schedules on the
-config by ``Loader.finds(Schedule)``.
+"""The main schedule running is `workflow_runner` function that trigger the
+multiprocess of `workflow_control` function for listing schedules on the
+config by `Loader.finds(Schedule)`.
 
-    The ``workflow_control`` is the scheduler function that release 2 schedule
-functions; ``workflow_task``, and ``workflow_monitor``.
+    The `workflow_control` is the scheduler function that release 2 schedule
+functions; `workflow_task`, and ``workflow_monitor``.
 
     ``workflow_control`` --- Every minute at :02 --> ``workflow_task``
 
@@ -34,7 +34,7 @@ from functools import partial, total_ordering
 from heapq import heappop, heappush
 from queue import Queue
 from textwrap import dedent
-from typing import Any, Optional
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.dataclasses import dataclass
@@ -49,10 +49,9 @@ from .cron import On
 from .exceptions import JobException, WorkflowException
 from .job import Job
 from .params import Param
-from .result import Result
+from .result import Result, Status
 from .templates import has_template, param2template
 from .utils import (
-    cut_id,
     gen_id,
     get_dt_now,
     reach_next_minute,
@@ -486,22 +485,25 @@ class Workflow(BaseModel):
         params: DictData,
         *,
         run_id: str | None = None,
-        log: type[Audit] = None,
+        parent_run_id: str | None = None,
+        audit: type[Audit] = None,
         queue: ReleaseQueue | None = None,
         override_log_name: str | None = None,
+        result: Result | None = None,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
         release templating that include logical date (release date), execution
         date, or running id to the params.
 
-            This method allow workflow use log object to save the execution
-        result to log destination like file log to the local `/logs` directory.
+            This method allow workflow use audit object to save the execution
+        result to audit destination like file audit to the local `/logs`
+        directory.
 
         Steps:
             - Initialize ReleaseQueue and Release if they do not pass.
             - Create release data for pass to parameter templating function.
             - Execute this workflow with mapping release data to its parameters.
-            - Writing result log
+            - Writing result audit
             - Remove this release on the running queue
             - Push this release to complete queue
 
@@ -509,17 +511,26 @@ class Workflow(BaseModel):
         :param params: A workflow parameter that pass to execute method.
         :param queue: A ReleaseQueue that use for mark complete.
         :param run_id: A workflow running ID for this release.
-        :param log: A log class that want to save the execution result.
+        :param parent_run_id: A parent workflow running ID for this release.
+        :param audit: An audit class that want to save the execution result.
         :param queue: A ReleaseQueue object.
         :param override_log_name: An override logging name that use instead
             the workflow name.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        log: type[Audit] = log or get_audit()
+        audit: type[Audit] = audit or get_audit()
         name: str = override_log_name or self.name
-        run_id: str = run_id or gen_id(name, unique=True)
-        rs_release: Result = Result(run_id=run_id)
+
+        if result is None:
+            result: Result = Result(
+                run_id=(run_id or gen_id(name, unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:  # pragma: no cov
+            result.set_parent_run_id(parent_run_id)
 
         if queue is not None and not isinstance(queue, ReleaseQueue):
             raise TypeError(
@@ -527,13 +538,11 @@ class Workflow(BaseModel):
             )
 
         # VALIDATE: Change release value to Release object.
-        rs_release_type: str = "release"
         if isinstance(release, datetime):
-            rs_release_type: str = "datetime"
             release: Release = Release.from_dt(release)
 
-        logger.debug(
-            f"({cut_id(run_id)}) [RELEASE]: Start release - {name!r} : "
+        result.trace.debug(
+            f"[RELEASE]: Start release - {name!r} : "
             f"{release.date:%Y-%m-%d %H:%M:%S}"
         )
 
@@ -543,34 +552,38 @@ class Workflow(BaseModel):
             "release": {
                 "logical_date": release.date,
                 "execute_date": datetime.now(tz=config.tz),
-                "run_id": run_id,
+                "run_id": result.run_id,
                 "timezone": config.tz,
             }
         }
 
         # NOTE: Execute workflow with templating params from release mapping.
-        rs: Result = self.execute(
+        #   The result context that return from execution method is:
+        #
+        #   ... {"params": ..., "jobs": ...}
+        #
+        self.execute(
             params=param2template(params, release_params),
-            run_id=run_id,
+            result=result,
+            parent_run_id=result.parent_run_id,
         )
-        logger.debug(
-            f"({cut_id(run_id)}) [RELEASE]: End release - {name!r} : "
+        result.trace.debug(
+            f"[RELEASE]: End release - {name!r} : "
             f"{release.date:%Y-%m-%d %H:%M:%S}"
         )
 
-        rs.set_parent_run_id(run_id)
-
-        # NOTE: Saving execution result to destination of the input log object.
-        logger.debug(f"({cut_id(run_id)}) [LOG]: Writing log: {name!r}.")
+        # NOTE: Saving execution result to destination of the input audit
+        #   object.
+        result.trace.debug(f"[LOG]: Writing audit: {name!r}.")
         (
-            log(
+            audit(
                 name=name,
                 release=release.date,
                 type=release.type,
-                context=rs.context,
-                parent_run_id=rs.parent_run_id,
-                run_id=rs.run_id,
-                execution_time=rs.alive_time(),
+                context=result.context,
+                parent_run_id=result.parent_run_id,
+                run_id=result.run_id,
+                execution_time=result.alive_time(),
             ).save(excluded=None)
         )
 
@@ -580,20 +593,25 @@ class Workflow(BaseModel):
             queue.mark_complete(release)
 
         # NOTE: Remove the params key from the result context for deduplicate.
-        context: dict[str, Any] = rs.context
-        context.pop("params")
+        #   This step is prepare result context for this release method.
+        context: DictData = result.context
+        jobs: DictData = context.pop("jobs", {})
+        errors: DictData = (
+            {"errors": context.pop("errors", {})} if "errors" in context else {}
+        )
 
-        return rs_release.catch(
-            status=0,
+        return result.catch(
+            status=Status.SUCCESS,
             context={
+                # NOTE: Update the real params that pass in this method.
                 "params": params,
                 "release": {
-                    "status": "success",
-                    "type": rs_release_type,
+                    "type": release.type,
                     "logical_date": release.date,
                     "release": release,
                 },
-                "outputs": context,
+                "outputs": {"jobs": jobs},
+                **errors,
             },
         )
 
@@ -602,7 +620,7 @@ class Workflow(BaseModel):
         offset: float,
         end_date: datetime,
         queue: ReleaseQueue,
-        log: type[Audit],
+        audit: type[Audit],
         *,
         force_run: bool = False,
     ) -> ReleaseQueue:
@@ -620,9 +638,9 @@ class Workflow(BaseModel):
         :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to make log object.
-        :param force_run: A flag that allow to release workflow if the log with
-            that release was pointed.
+        :param audit: An audit class that want to make audit object.
+        :param force_run: A flag that allow to release workflow if the audit
+            with that release was pointed.
 
         :rtype: ReleaseQueue
         """
@@ -645,7 +663,7 @@ class Workflow(BaseModel):
             )
 
             while queue.check_queue(workflow_release) or (
-                log.is_pointed(name=self.name, release=workflow_release.date)
+                audit.is_pointed(name=self.name, release=workflow_release.date)
                 and not force_run
             ):
                 workflow_release = Release(
@@ -671,16 +689,16 @@ class Workflow(BaseModel):
         *,
         run_id: str | None = None,
         periods: int = 1,
-        log: Audit | None = None,
+        audit: Audit | None = None,
         force_run: bool = False,
         timeout: int = 1800,
-    ) -> list[Result]:
-        """Poke this workflow with start datetime value that passing to its
-        ``on`` field with threading executor pool for executing with all its
-        schedules that was set on the `on` value.
+    ) -> Result:
+        """Poke function with a start datetime value that will pass to its
+        `on` field on the threading executor pool for execute the `release`
+        method (It run all schedules that was set on the `on` values).
 
             This method will observe its schedule that nearing to run with the
-        ``self.release()`` method.
+        `self.release()` method.
 
             The limitation of this method is not allow run a date that less
         than the current date.
@@ -689,17 +707,19 @@ class Workflow(BaseModel):
         :param params: A parameters that want to pass to the release method.
         :param run_id: A workflow running ID for this poke.
         :param periods: A periods in minutes value that use to run this poking.
-        :param log: A log object that want to use on this poking process.
-        :param force_run: A flag that allow to release workflow if the log with
+        :param audit: An audit object that want to use on this poking process.
+        :param force_run: A flag that allow to release workflow if the audit with
             that release was pointed.
         :param timeout: A second value for timeout while waiting all futures
             run completely.
 
-        :rtype: list[Result]
-        :return: A list of all results that return from ``self.release`` method.
+        :rtype: Result
+        :return: A list of all results that return from `self.release` method.
         """
-        log: type[Audit] = log or get_audit()
-        run_id: str = run_id or gen_id(self.name, unique=True)
+        audit: type[Audit] = audit or get_audit()
+        result: Result = Result(
+            run_id=(run_id or gen_id(self.name, unique=True))
+        )
 
         # VALIDATE: Check the periods value should gather than 0.
         if periods <= 0:
@@ -710,11 +730,10 @@ class Workflow(BaseModel):
         # NOTE: If this workflow does not set the on schedule, it will return
         #   empty result.
         if len(self.on) == 0:
-            logger.info(
-                f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have any "
-                f"schedule to run."
+            result.trace.info(
+                f"[POKING]: {self.name!r} does not have any schedule to run."
             )
-            return []
+            return result.catch(status=Status.SUCCESS, context={"outputs": []})
 
         # NOTE: Create the current date that change microsecond to 0
         current_date: datetime = datetime.now(tz=config.tz).replace(
@@ -737,13 +756,13 @@ class Workflow(BaseModel):
         #   periods value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
-        logger.info(
-            f"({cut_id(run_id)}) [POKING]: Start Poking: {self.name!r} from "
+        result.trace.info(
+            f"[POKING]: Start Poking: {self.name!r} from "
             f"{start_date:%Y-%m-%d %H:%M:%S} to {end_date:%Y-%m-%d %H:%M:%S}"
         )
 
         params: DictData = {} if params is None else params
-        results: list[Result] = []
+        context: list[Result] = []
 
         # NOTE: Create empty ReleaseQueue object.
         q: ReleaseQueue = ReleaseQueue()
@@ -751,17 +770,16 @@ class Workflow(BaseModel):
         # NOTE: Create reusable partial function and add Release to the release
         #   queue object.
         partial_queue = partial(
-            self.queue, offset, end_date, log=log, force_run=force_run
+            self.queue, offset, end_date, audit=audit, force_run=force_run
         )
         partial_queue(q)
 
         # NOTE: Return the empty result if it does not have any Release.
         if not q.is_queued:
-            logger.info(
-                f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have "
-                f"any queue."
+            result.trace.info(
+                f"[POKING]: {self.name!r} does not have any queue."
             )
-            return []
+            return result.catch(status=Status.SUCCESS, context={"outputs": []})
 
         # NOTE: Start create the thread pool executor for running this poke
         #   process.
@@ -778,8 +796,8 @@ class Workflow(BaseModel):
                 release: Release = heappop(q.queue)
 
                 if reach_next_minute(release.date, tz=config.tz, offset=offset):
-                    logger.debug(
-                        f"({cut_id(run_id)}) [POKING]: The latest release, "
+                    result.trace.debug(
+                        f"[POKING]: The latest release, "
                         f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
                         f"on this minute"
                     )
@@ -799,8 +817,9 @@ class Workflow(BaseModel):
                         self.release,
                         release=release,
                         params=params,
-                        log=log,
+                        audit=audit,
                         queue=q,
+                        parent_run_id=result.run_id,
                     )
                 )
 
@@ -809,18 +828,20 @@ class Workflow(BaseModel):
             # WARNING: This poking method does not allow to use fail-fast
             #   logic to catching parallel execution result.
             for future in as_completed(futures, timeout=timeout):
-                results.append(future.result().set_parent_run_id(run_id))
+                context.append(future.result())
 
-        return results
+        return result.catch(
+            status=Status.SUCCESS,
+            context={"outputs": context},
+        )
 
     def execute_job(
         self,
         job_id: str,
         params: DictData,
         *,
-        run_id: str | None = None,
-        raise_error: bool = True,
         result: Result | None = None,
+        raise_error: bool = True,
     ) -> Result:
         """Job execution with passing dynamic parameters from the main workflow
         execution to the target job object via job's ID.
@@ -835,7 +856,6 @@ class Workflow(BaseModel):
 
         :param job_id: A job ID that want to execute.
         :param params: A params that was parameterized from workflow execution.
-        :param run_id: A workflow running ID for this job execution.
         :param raise_error: A flag that raise error instead catching to result
             if it gets exception from job execution.
         :param result: (Result) A result object for keeping context and status
@@ -846,10 +866,7 @@ class Workflow(BaseModel):
             context.
         """
         if result is None:  # pragma: no cov
-            run_id: str = run_id or gen_id(self.name, unique=True)
-            result: Result = Result(run_id=run_id)
-        else:
-            run_id: str = result.run_id
+            result: Result = Result(run_id=gen_id(self.name, unique=True))
 
         # VALIDATE: check a job ID that exists in this workflow or not.
         if job_id not in self.jobs:
@@ -862,13 +879,18 @@ class Workflow(BaseModel):
 
         # IMPORTANT:
         #   This execution change all job running IDs to the current workflow
-        #   execution running ID (with passing run_id to the job execution
-        #   argument).
+        #   running ID, but it still trac log to the same parent running ID
+        #   (with passing `run_id` and `parent_run_id` to the job execution
+        #   arguments).
         #
         try:
             job: Job = self.jobs[job_id]
             job.set_outputs(
-                job.execute(params=params, run_id=run_id).context,
+                job.execute(
+                    params=params,
+                    run_id=result.run_id,
+                    parent_run_id=result.parent_run_id,
+                ).context,
                 to=params,
             )
         except JobException as err:
@@ -888,6 +910,7 @@ class Workflow(BaseModel):
         params: DictData,
         *,
         run_id: str | None = None,
+        parent_run_id: str | None = None,
         timeout: int = 0,
         result: Result | None = None,
     ) -> Result:
@@ -902,37 +925,36 @@ class Workflow(BaseModel):
         can access it with syntax:
 
             ... ${job-name}.stages.${stage-id}.outputs.${key}
+            ... ${job-name}.stages.${stage-id}.errors.${key}
 
         :param params: An input parameters that use on workflow execution that
             will parameterize before using it. Default is None.
-        :type params: DictData
-
         :param run_id: A workflow running ID for this job execution.
-        :type run_id: str | None (default: None)
-        :param timeout: (int) A workflow execution time out in second unit that use
-            for limit time of execution and waiting job dependency. This value
-            does not force stop the task that still running more than this limit
-            time. (default: 0)
+        :param parent_run_id: A parent workflow running ID for this release.
+        :param timeout: (int) A workflow execution time out in second unit that
+            use for limit time of execution and waiting job dependency. This
+            value does not force stop the task that still running more than this
+            limit time. (default: 0)
         :param result: (Result) A result object for keeping context and status
             data.
 
         :rtype: Result
         """
-        # NOTE: I use this condition because this method allow passing empty
-        #   params and I do not want to create new dict object.
         ts: float = time.monotonic()
         if result is None:  # pragma: no cov
             result: Result = Result(
-                run_id=(run_id or gen_id(self.name, unique=True))
+                run_id=(run_id or gen_id(self.name, unique=True)),
+                parent_run_id=parent_run_id,
             )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
 
         result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
 
         # NOTE: It should not do anything if it does not have job.
         if not self.jobs:
             result.trace.warning(
-                f"[WORKFLOW]: This workflow: {self.name!r} does not have any "
-                f"jobs"
+                f"[WORKFLOW]: Workflow: {self.name!r} does not have any jobs"
             )
             return result.catch(status=0, context=params)
 
@@ -973,8 +995,11 @@ class Workflow(BaseModel):
             status: int = 1
             context.update(
                 {
-                    "error": err,
-                    "error_message": f"{err.__class__.__name__}: {err}",
+                    "errors": {
+                        "class": err,
+                        "name": err.__class__.__name__,
+                        "message": f"{err.__class__.__name__}: {err}",
+                    },
                 },
             )
         return result.catch(status=status, context=context)
@@ -1041,7 +1066,7 @@ class Workflow(BaseModel):
                 futures.append(
                     executor.submit(
                         self.execute_job,
-                        job_id,
+                        job_id=job_id,
                         params=context,
                         result=result,
                     ),
@@ -1070,9 +1095,7 @@ class Workflow(BaseModel):
                 future.cancel()
 
         # NOTE: Raise timeout error.
-        result.trace.warning(
-            f"[WORKFLOW]: Execution: {self.name!r} was timeout."
-        )
+        result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
     def __exec_non_threading(
@@ -1137,9 +1160,7 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        result.trace.warning(
-            f"[WORKFLOW]: Execution: {self.name!r} was timeout."
-        )
+        result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
 
@@ -1151,8 +1172,8 @@ class WorkflowTask:
         This dataclass object is mapping 1-to-1 with workflow and cron runner
     objects.
 
-        This dataclass has the release method for itself.
-
+        This dataclass has the release method for itself that prepare necessary
+    arguments before passing to the parent release method.
     """
 
     alias: str
@@ -1164,19 +1185,23 @@ class WorkflowTask:
         self,
         release: datetime | Release | None = None,
         run_id: str | None = None,
-        log: type[Audit] = None,
+        audit: type[Audit] = None,
         queue: ReleaseQueue | None = None,
     ) -> Result:
-        """Release the workflow task data.
+        """Release the workflow task that passing an override parameter to
+        the parent release method with the `values` field.
+
+            This method can handler not passing release value by default
+        generate step. It uses the `runner` field for generate release object.
 
         :param release: A release datetime or Release object.
         :param run_id: A workflow running ID for this release.
-        :param log: A log class that want to save the execution result.
+        :param audit: An audit class that want to save the execution result.
         :param queue: A ReleaseQueue object that use to mark complete.
 
         :rtype: Result
         """
-        log: type[Audit] = log or get_audit()
+        audit: type[Audit] = audit or get_audit()
 
         if release is None:
 
@@ -1203,7 +1228,7 @@ class WorkflowTask:
             release=release,
             params=self.values,
             run_id=run_id,
-            log=log,
+            audit=audit,
             queue=queue,
             override_log_name=self.alias,
         )
@@ -1212,7 +1237,7 @@ class WorkflowTask:
         self,
         end_date: datetime,
         queue: ReleaseQueue,
-        log: type[Audit],
+        audit: type[Audit],
         *,
         force_run: bool = False,
     ) -> ReleaseQueue:
@@ -1221,9 +1246,9 @@ class WorkflowTask:
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to make log object.
+        :param audit: An audit class that want to make audit object.
         :param force_run: (bool) A flag that allow to release workflow if the
-            log with that release was pointed.
+            audit with that release was pointed.
 
         :rtype: ReleaseQueue
         """
@@ -1239,7 +1264,7 @@ class WorkflowTask:
         )
 
         while queue.check_queue(workflow_release) or (
-            log.is_pointed(name=self.alias, release=workflow_release.date)
+            audit.is_pointed(name=self.alias, release=workflow_release.date)
             and not force_run
         ):
             workflow_release = Release(