ddeutil-workflow 0.0.33__py3-none-any.whl → 0.0.34__py3-none-any.whl

ddeutil/workflow/job.py

@@ -39,7 +39,7 @@ from .exceptions import (
     UtilException,
 )
 from .result import Result, Status
-from .stage import Stage
+from .stages import Stage
 from .templates import has_template
 from .utils import (
     cross_product,
@@ -400,10 +400,15 @@ class Job(BaseModel):
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
+
+        errors: DictData = (
+            {"errors": output.pop("errors", {})} if "errors" in output else {}
+        )
+
         to["jobs"][_id] = (
-            {"strategies": output}
+            {"strategies": output, **errors}
             if self.strategy.is_set()
-            else output.get(next(iter(output), "DUMMY"), {})
+            else {**output.get(next(iter(output), "DUMMY"), {}), **errors}
         )
         return to
 
@@ -412,7 +417,6 @@ class Job(BaseModel):
         strategy: DictData,
         params: DictData,
         *,
-        run_id: str | None = None,
         result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
@@ -432,7 +436,6 @@ class Job(BaseModel):
         :param strategy: A strategy metrix value that use on this execution.
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
-        :param run_id: A job running ID for this strategy execution.
         :param result: (Result) A result object for keeping context and status
             data.
         :param event: An event manager that pass to the PoolThreadExecutor.
@@ -440,9 +443,7 @@ class Job(BaseModel):
         :rtype: Result
         """
         if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(run_id or gen_id(self.id or "", unique=True))
-            )
+            result: Result = Result(run_id=gen_id(self.id or "", unique=True))
 
         strategy_id: str = gen_id(strategy)
 
@@ -492,8 +493,11 @@ class Job(BaseModel):
                             # "stages": filter_func(context.pop("stages", {})),
                             #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(error_msg),
-                            "error_message": error_msg,
+                            "errors": {
+                                "class": JobException(error_msg),
+                                "name": "JobException",
+                                "message": error_msg,
+                            },
                         },
                     },
                 )
@@ -516,10 +520,18 @@ class Job(BaseModel):
             #       "stages": { { "stage-id-1": ... }, ... }
             #   }
             #
+            # IMPORTANT:
+            #   This execution change all stage running IDs to the current job
+            #   running ID, but it still trac log to the same parent running ID
+            #   (with passing `run_id` and `parent_run_id` to the stage
+            #   execution arguments).
+            #
             try:
                 stage.set_outputs(
                     stage.handler_execute(
-                        params=context, run_id=result.run_id
+                        params=context,
+                        run_id=result.run_id,
+                        parent_run_id=result.parent_run_id,
                     ).context,
                     to=context,
                 )
@@ -527,17 +539,21 @@ class Job(BaseModel):
                 result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
                 if config.job_raise_error:
                     raise JobException(
-                        f"Get stage execution error: {err.__class__.__name__}: "
+                        f"Stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
+
                 return result.catch(
                     status=1,
                     context={
                         strategy_id: {
                             "matrix": strategy,
                             "stages": context.pop("stages", {}),
-                            "error": err,
-                            "error_message": f"{err.__class__.__name__}: {err}",
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
                         },
                     },
                 )
@@ -560,6 +576,7 @@ class Job(BaseModel):
         params: DictData,
         *,
         run_id: str | None = None,
+        parent_run_id: str | None = None,
         result: Result | None = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
@@ -568,6 +585,7 @@ class Job(BaseModel):
 
         :param params: An input parameters that use on job execution.
         :param run_id: A job running ID for this execution.
+        :param parent_run_id: A parent workflow running ID for this release.
         :param result: (Result) A result object for keeping context and status
             data.
 
@@ -577,8 +595,12 @@ class Job(BaseModel):
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
         if result is None:  # pragma: no cov
-            run_id: str = run_id or gen_id(self.id or "", unique=True)
-            result: Result = Result(run_id=run_id)
+            result: Result = Result(
+                run_id=(run_id or gen_id(self.id or "", unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
 
         # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
@@ -614,110 +636,50 @@ class Job(BaseModel):
                 for strategy in self.strategy.make()
             ]
 
-            return (
-                self.__catch_fail_fast(event, futures=futures, result=result)
-                if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures, result=result)
-            )
-
-    @staticmethod
-    def __catch_fail_fast(
-        event: Event,
-        futures: list[Future],
-        result: Result,
-        *,
-        timeout: int = 1800,
-    ) -> Result:
-        """Job parallel pool futures catching with fail-fast mode. That will
-        stop and set event on all not done futures if it receives the first
-        exception from all running futures.
-
-        :param event: An event manager instance that able to set stopper on the
-            observing multithreading.
-        :param futures: A list of futures.
-        :param result: (Result) A result object for keeping context and status
-            data.
-        :param timeout: A timeout to waiting all futures complete.
-
-        :rtype: Result
-        """
-        context: DictData = {}
-        status: Status = Status.SUCCESS
-
-        # NOTE: Get results from a collection of tasks with a timeout that has
-        #   the first exception.
-        done, not_done = wait(
-            futures, timeout=timeout, return_when=FIRST_EXCEPTION
-        )
-        nd: str = (
-            f", the strategies do not run is {not_done}" if not_done else ""
-        )
-        result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
-
-        # NOTE:
-        #       Stop all running tasks with setting the event manager and cancel
-        #   any scheduled tasks.
-        #
-        if len(done) != len(futures):
-            event.set()
-            for future in not_done:
-                future.cancel()
-
-        future: Future
-        for future in done:
-
-            # NOTE: Handle the first exception from feature
-            if err := future.exception():
-                status: Status = Status.FAILED
-                result.trace.error(
-                    f"[JOB]: Fail-fast catching:\n\t{future.exception()}"
-                )
-                context.update(
-                    {
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
-                )
-                continue
-
-            # NOTE: Update the result context to main job context.
-            future.result()
-
-        return result.catch(status=status, context=context)
-
-    @staticmethod
-    def __catch_all_completed(
-        futures: list[Future],
-        result: Result,
-        *,
-        timeout: int = 1800,
-    ) -> Result:
-        """Job parallel pool futures catching with all-completed mode.
-
-        :param futures: A list of futures.
-        :param result: (Result) A result object for keeping context and status
-            data.
-        :param timeout: A timeout to waiting all futures complete.
+            context: DictData = {}
+            status: Status = Status.SUCCESS
+            fail_fast_flag: bool = self.strategy.fail_fast
 
-        :rtype: Result
-        """
-        context: DictData = {}
-        status: Status = Status.SUCCESS
-
-        for future in as_completed(futures, timeout=timeout):
-            try:
-                future.result()
-            except JobException as err:
-                status = Status.FAILED
-                result.trace.error(
-                    f"[JOB]: All-completed catching:\n\t"
-                    f"{err.__class__.__name__}:\n\t{err}"
+            if fail_fast_flag:
+                # NOTE: Get results from a collection of tasks with a timeout
+                #   that has the first exception.
+                done, not_done = wait(
+                    futures, timeout=1800, return_when=FIRST_EXCEPTION
                 )
-                context.update(
-                    {
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
+                nd: str = (
+                    f", the strategies do not run is {not_done}"
+                    if not_done
+                    else ""
                 )
+                result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
+                # NOTE: Stop all running tasks with setting the event manager
+                #   and cancel any scheduled tasks.
+                if len(done) != len(futures):
+                    event.set()
+                    for future in not_done:
+                        future.cancel()
+            else:
+                done = as_completed(futures, timeout=1800)
+
+            for future in done:
+                try:
+                    future.result()
+                except JobException as err:
+                    status = Status.FAILED
+                    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
+                    result.trace.error(
+                        f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
+                        f"\n\t{err}"
+                    )
+                    context.update(
+                        {
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
+                        },
+                    )
 
         return result.catch(status=status, context=context)

ddeutil/workflow/result.py

@@ -3,12 +3,19 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""This is the Result module. It is the data context transfer objects that use
+by all object in this package.
+"""
 from __future__ import annotations
 
+import os
+from abc import ABC, abstractmethod
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from threading import Event
+from inspect import Traceback, currentframe, getframeinfo
+from pathlib import Path
+from threading import Event, get_ident
 from typing import Optional
 
 from pydantic import ConfigDict
@@ -19,11 +26,14 @@ from .__types import DictData, TupleStr
 from .conf import config, get_logger
 from .utils import cut_id, gen_id, get_dt_now
 
-logger = get_logger("ddeutil.workflow.audit")
+logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
     "Result",
     "Status",
+    "TraceLog",
+    "default_gen_id",
+    "get_dt_tznow",
 )
 
 
@@ -52,25 +62,109 @@ class Status(IntEnum):
     WAIT: int = 2
 
 
-class TraceLog:  # pragma: no cov
-    """Trace Log object."""
+@dataclass(frozen=True)
+class BaseTraceLog(ABC):  # pragma: no cov
+    """Base Trace Log dataclass object."""
 
-    __slots__: TupleStr = ("run_id",)
+    run_id: str
+    parent_run_id: Optional[str] = None
 
-    def __init__(self, run_id: str):
-        self.run_id: str = run_id
+    @abstractmethod
+    def writer(self, message: str, is_err: bool = False) -> None: ...
+
+    @abstractmethod
+    def make_message(self, message: str) -> str: ...
 
     def debug(self, message: str):
-        logger.debug(f"({cut_id(self.run_id)}) {message}")
+        msg: str = self.make_message(message)
+
+        # NOTE: Write file if debug mode.
+        if config.debug:
+            self.writer(msg)
+
+        logger.debug(msg, stacklevel=2)
 
     def info(self, message: str):
-        logger.info(f"({cut_id(self.run_id)}) {message}")
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.info(msg, stacklevel=2)
 
     def warning(self, message: str):
-        logger.warning(f"({cut_id(self.run_id)}) {message}")
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.warning(msg, stacklevel=2)
 
     def error(self, message: str):
-        logger.error(f"({cut_id(self.run_id)}) {message}")
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.error(msg, stacklevel=2)
+
+
+class TraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write file to the local storage."""
+
+    @property
+    def log_file(self) -> Path:
+        log_file: Path = (
+            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+        )
+        if not log_file.exists():
+            log_file.mkdir(parents=True)
+        return log_file
+
+    @property
+    def cut_id(self) -> str:
+        """Combine cutting ID of parent running ID if it set."""
+        cut_run_id: str = cut_id(self.run_id)
+        if not self.parent_run_id:
+            return f"{cut_run_id} -> {' ' * 6}"
+
+        cut_parent_run_id: str = cut_id(self.parent_run_id)
+        return f"{cut_parent_run_id} -> {cut_run_id}"
+
+    def make_message(self, message: str) -> str:
+        return f"({self.cut_id}) {message}"
+
+    def writer(self, message: str, is_err: bool = False) -> None:
+        """The path of logging data will store by format:
+
+            ... ./logs/run_id=<run-id>/stdout.txt
+            ... ./logs/run_id=<run-id>/stderr.txt
+
+        :param message:
+        :param is_err:
+        """
+        if not config.enable_write_log:
+            return
+
+        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
+        filename: str = frame_info.filename.split(os.path.sep)[-1]
+        lineno: int = frame_info.lineno
+
+        # NOTE: set process and thread IDs.
+        process: int = os.getpid()
+        thread: int = get_ident()
+
+        write_file: str = "stderr.txt" if is_err else "stdout.txt"
+        with (self.log_file / write_file).open(
+            mode="at", encoding="utf-8"
+        ) as f:
+            msg_fmt: str = f"{config.log_format_file}\n"
+            print(msg_fmt)
+            f.write(
+                msg_fmt.format(
+                    **{
+                        "datetime": get_dt_tznow().strftime(
+                            config.log_datetime_format
+                        ),
+                        "process": process,
+                        "thread": thread,
+                        "message": message,
+                        "filename": filename,
+                        "lineno": lineno,
+                    }
+                )
+            )
 
 
 @dataclass(
@@ -94,6 +188,26 @@ class Result:
     event: Event = field(default_factory=Event, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
+    @classmethod
+    def construct_with_rs_or_id(
+        cls,
+        result: Result | None = None,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        id_logic: str | None = None,
+    ) -> Self:  # pragma: no cov
+        """Create the Result object or set parent running id if passing Result
+        object.
+        """
+        if result is None:
+            result: Result = cls(
+                run_id=(run_id or gen_id(id_logic or "", unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
+        return result
+
     def set_run_id(self, running_id: str) -> Self:
         """Set a running ID.
 
@@ -130,26 +244,13 @@ class Result:
         self.__dict__["context"].update(context or {})
         return self
 
-    def receive(self, result: Result) -> Self:
-        """Receive context from another result object.
-
-        :rtype: Self
-        """
-        self.__dict__["status"] = result.status
-        self.__dict__["context"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id = result.parent_run_id
-        self.run_id = result.run_id
-        return self
-
     @property
     def trace(self) -> TraceLog:
         """Return TraceLog object that passing its running ID.
 
         :rtype: TraceLog
         """
-        return TraceLog(self.run_id)
+        return TraceLog(self.run_id, self.parent_run_id)
 
     def alive_time(self) -> float:  # pragma: no cov
         return (get_dt_tznow() - self.ts).total_seconds()