ddeutil-workflow 0.0.32__py3-none-any.whl → 0.0.34__py3-none-any.whl

ddeutil/workflow/job.py

@@ -38,12 +38,11 @@ from .exceptions import (
     StageException,
     UtilException,
 )
-from .result import Result
-from .stage import Stage
+from .result import Result, Status
+from .stages import Stage
 from .templates import has_template
 from .utils import (
     cross_product,
-    cut_id,
     dash2underscore,
     filter_func,
     gen_id,
@@ -222,6 +221,8 @@ class RunsOn(str, Enum):
 
     local: str = "local"
     docker: str = "docker"
+    self_hosted: str = "self_hosted"
+    k8s: str = "k8s"
 
 
 class Job(BaseModel):
@@ -399,10 +400,15 @@ class Job(BaseModel):
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
+
+        errors: DictData = (
+            {"errors": output.pop("errors", {})} if "errors" in output else {}
+        )
+
         to["jobs"][_id] = (
-            {"strategies": output}
+            {"strategies": output, **errors}
             if self.strategy.is_set()
-            else output.get(next(iter(output), "DUMMY"), {})
+            else {**output.get(next(iter(output), "DUMMY"), {}), **errors}
         )
         return to
 
@@ -411,7 +417,7 @@ class Job(BaseModel):
         strategy: DictData,
         params: DictData,
         *,
-        run_id: str | None = None,
+        result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
@@ -430,14 +436,16 @@ class Job(BaseModel):
         :param strategy: A strategy metrix value that use on this execution.
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
-        :param run_id: A job running ID for this strategy execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :param event: An event manager that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
-        run_id: str = run_id or gen_id(self.id or "", unique=True)
+        if result is None:  # pragma: no cov
+            result: Result = Result(run_id=gen_id(self.id or "", unique=True))
+
         strategy_id: str = gen_id(strategy)
-        rs: Result = Result(run_id=run_id)
 
         # PARAGRAPH:
         #
@@ -458,18 +466,14 @@ class Job(BaseModel):
         for stage in self.stages:
 
             if stage.is_skipped(params=context):
-                logger.info(
-                    f"({cut_id(run_id)}) [JOB]: Skip stage: {stage.iden!r}"
-                )
+                result.trace.info(f"[JOB]: Skip stage: {stage.iden!r}")
                 continue
 
-            logger.info(
-                f"({cut_id(run_id)}) [JOB]: Execute stage: {stage.iden!r}"
-            )
+            result.trace.info(f"[JOB]: Execute stage: {stage.iden!r}")
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({cut_id(run_id)}) [JOB]: ... Matrix: {strategy}")
+                result.trace.info(f"[JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -478,7 +482,7 @@ class Job(BaseModel):
                     "Job strategy was canceled from event that had set before "
                     "strategy execution."
                 )
-                return rs.catch(
+                return result.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -489,8 +493,11 @@ class Job(BaseModel):
                             # "stages": filter_func(context.pop("stages", {})),
                             #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(error_msg),
-                            "error_message": error_msg,
+                            "errors": {
+                                "class": JobException(error_msg),
+                                "name": "JobException",
+                                "message": error_msg,
+                            },
                         },
                     },
                 )
@@ -513,30 +520,40 @@ class Job(BaseModel):
             #       "stages": { { "stage-id-1": ... }, ... }
             #   }
             #
+            # IMPORTANT:
+            #   This execution change all stage running IDs to the current job
+            #   running ID, but it still trac log to the same parent running ID
+            #   (with passing `run_id` and `parent_run_id` to the stage
+            #   execution arguments).
+            #
             try:
                 stage.set_outputs(
                     stage.handler_execute(
-                        params=context, run_id=run_id
+                        params=context,
+                        run_id=result.run_id,
+                        parent_run_id=result.parent_run_id,
                     ).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
-                logger.error(
-                    f"({cut_id(run_id)}) [JOB]: {err.__class__.__name__}: {err}"
-                )
+                result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
                 if config.job_raise_error:
                     raise JobException(
-                        f"Get stage execution error: {err.__class__.__name__}: "
+                        f"Stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                return rs.catch(
+
+                return result.catch(
                     status=1,
                     context={
                         strategy_id: {
                             "matrix": strategy,
                             "stages": context.pop("stages", {}),
-                            "error": err,
-                            "error_message": f"{err.__class__.__name__}: {err}",
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
                         },
                     },
                 )
@@ -544,8 +561,8 @@ class Job(BaseModel):
             # NOTE: Remove the current stage object for saving memory.
             del stage
 
-        return rs.catch(
-            status=0,
+        return result.catch(
+            status=Status.SUCCESS,
             context={
                 strategy_id: {
                     "matrix": strategy,
@@ -554,36 +571,49 @@ class Job(BaseModel):
             },
         )
 
-    def execute(self, params: DictData, run_id: str | None = None) -> Result:
+    def execute(
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        result: Result | None = None,
+    ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the ``stages`` field of this job.
 
         :param params: An input parameters that use on job execution.
         :param run_id: A job running ID for this execution.
+        :param parent_run_id: A parent workflow running ID for this release.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
 
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
-        run_id: str = run_id or gen_id(self.id or "", unique=True)
-        context: DictData = {}
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=(run_id or gen_id(self.id or "", unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
 
         # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
+
             for strategy in self.strategy.make():
-                rs: Result = self.execute_strategy(
+                result: Result = self.execute_strategy(
                     strategy=strategy,
                     params=params,
-                    run_id=run_id,
+                    result=result,
                 )
-                context.update(rs.context)
-            return Result(
-                status=0,
-                context=context,
-            )
+
+            return result.catch(status=Status.SUCCESS)
 
         # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
@@ -600,117 +630,56 @@ class Job(BaseModel):
                     self.execute_strategy,
                     strategy=strategy,
                     params=params,
-                    run_id=run_id,
+                    result=result,
                     event=event,
                 )
                 for strategy in self.strategy.make()
             ]
 
-            return (
-                self.__catch_fail_fast(event, futures=futures, run_id=run_id)
-                if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures, run_id=run_id)
-            )
-
-    @staticmethod
-    def __catch_fail_fast(
-        event: Event,
-        futures: list[Future],
-        run_id: str,
-        *,
-        timeout: int = 1800,
-    ) -> Result:
-        """Job parallel pool futures catching with fail-fast mode. That will
-        stop and set event on all not done futures if it receives the first
-        exception from all running futures.
-
-        :param event: An event manager instance that able to set stopper on the
-            observing multithreading.
-        :param futures: A list of futures.
-        :param run_id: A job running ID from execution.
-        :param timeout: A timeout to waiting all futures complete.
+            context: DictData = {}
+            status: Status = Status.SUCCESS
+            fail_fast_flag: bool = self.strategy.fail_fast
 
-        :rtype: Result
-        """
-        rs_final: Result = Result(run_id=run_id)
-        context: DictData = {}
-        status: int = 0
-
-        # NOTE: Get results from a collection of tasks with a timeout that has
-        #   the first exception.
-        done, not_done = wait(
-            futures, timeout=timeout, return_when=FIRST_EXCEPTION
-        )
-        nd: str = (
-            f", the strategies do not run is {not_done}" if not_done else ""
-        )
-        logger.debug(f"({cut_id(run_id)}) [JOB]: Strategy is set Fail Fast{nd}")
-
-        # NOTE:
-        #       Stop all running tasks with setting the event manager and cancel
-        #   any scheduled tasks.
-        #
-        if len(done) != len(futures):
-            event.set()
-            for future in not_done:
-                future.cancel()
-
-        future: Future
-        for future in done:
-
-            # NOTE: Handle the first exception from feature
-            if err := future.exception():
-                status: int = 1
-                logger.error(
-                    f"({cut_id(run_id)}) [JOB]: Fail-fast catching:\n\t"
-                    f"{future.exception()}"
+            if fail_fast_flag:
+                # NOTE: Get results from a collection of tasks with a timeout
+                #   that has the first exception.
+                done, not_done = wait(
+                    futures, timeout=1800, return_when=FIRST_EXCEPTION
                 )
-                context.update(
-                    {
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
-                )
-                continue
-
-            # NOTE: Update the result context to main job context.
-            context.update(future.result().context)
-
-        return rs_final.catch(status=status, context=context)
-
-    @staticmethod
-    def __catch_all_completed(
-        futures: list[Future],
-        run_id: str,
-        *,
-        timeout: int = 1800,
-    ) -> Result:
-        """Job parallel pool futures catching with all-completed mode.
-
-        :param futures: A list of futures.
-        :param run_id: A job running ID from execution.
-        :param timeout: A timeout to waiting all futures complete.
-
-        :rtype: Result
-        """
-        rs_final: Result = Result(run_id=run_id)
-        context: DictData = {}
-        status: int = 0
-
-        for future in as_completed(futures, timeout=timeout):
-            try:
-                context.update(future.result().context)
-            except JobException as err:
-                status = 1
-                logger.error(
-                    f"({cut_id(run_id)}) [JOB]: All-completed catching:\n\t"
-                    f"{err.__class__.__name__}:\n\t{err}"
-                )
-                context.update(
-                    {
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
+                nd: str = (
+                    f", the strategies do not run is {not_done}"
+                    if not_done
+                    else ""
                 )
+                result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
+                # NOTE: Stop all running tasks with setting the event manager
+                #   and cancel any scheduled tasks.
+                if len(done) != len(futures):
+                    event.set()
+                    for future in not_done:
+                        future.cancel()
+            else:
+                done = as_completed(futures, timeout=1800)
+
+            for future in done:
+                try:
+                    future.result()
+                except JobException as err:
+                    status = Status.FAILED
+                    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
+                    result.trace.error(
+                        f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
+                        f"\n\t{err}"
+                    )
+                    context.update(
+                        {
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
+                        },
+                    )
 
-        return rs_final.catch(status=status, context=context)
+        return result.catch(status=status, context=context)

ddeutil/workflow/result.py

@@ -3,22 +3,173 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""This is the Result module. It is the data context transfer objects that use
+by all object in this package.
+"""
 from __future__ import annotations
 
+import os
+from abc import ABC, abstractmethod
 from dataclasses import field
+from datetime import datetime
+from enum import IntEnum
+from inspect import Traceback, currentframe, getframeinfo
+from pathlib import Path
+from threading import Event, get_ident
 from typing import Optional
 
+from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
-from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .utils import gen_id
+from .conf import config, get_logger
+from .utils import cut_id, gen_id, get_dt_now
 
-__all__: TupleStr = ("Result",)
+logger = get_logger("ddeutil.workflow")
 
+__all__: TupleStr = (
+    "Result",
+    "Status",
+    "TraceLog",
+    "default_gen_id",
+    "get_dt_tznow",
+)
 
-@dataclass
+
+def default_gen_id() -> str:
+    """Return running ID which use for making default ID for the Result model if
+    a run_id field initializes at the first time.
+
+    :rtype: str
+    """
+    return gen_id("manual", unique=True)
+
+
+def get_dt_tznow() -> datetime:
+    """Return the current datetime object that passing the config timezone.
+
+    :rtype: datetime
+    """
+    return get_dt_now(tz=config.tz)
+
+
+class Status(IntEnum):
+    """Status Int Enum object."""
+
+    SUCCESS: int = 0
+    FAILED: int = 1
+    WAIT: int = 2
+
+
+@dataclass(frozen=True)
+class BaseTraceLog(ABC):  # pragma: no cov
+    """Base Trace Log dataclass object."""
+
+    run_id: str
+    parent_run_id: Optional[str] = None
+
+    @abstractmethod
+    def writer(self, message: str, is_err: bool = False) -> None: ...
+
+    @abstractmethod
+    def make_message(self, message: str) -> str: ...
+
+    def debug(self, message: str):
+        msg: str = self.make_message(message)
+
+        # NOTE: Write file if debug mode.
+        if config.debug:
+            self.writer(msg)
+
+        logger.debug(msg, stacklevel=2)
+
+    def info(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.info(msg, stacklevel=2)
+
+    def warning(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.warning(msg, stacklevel=2)
+
+    def error(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.error(msg, stacklevel=2)
+
+
+class TraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write file to the local storage."""
+
+    @property
+    def log_file(self) -> Path:
+        log_file: Path = (
+            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+        )
+        if not log_file.exists():
+            log_file.mkdir(parents=True)
+        return log_file
+
+    @property
+    def cut_id(self) -> str:
+        """Combine cutting ID of parent running ID if it set."""
+        cut_run_id: str = cut_id(self.run_id)
+        if not self.parent_run_id:
+            return f"{cut_run_id} -> {' ' * 6}"
+
+        cut_parent_run_id: str = cut_id(self.parent_run_id)
+        return f"{cut_parent_run_id} -> {cut_run_id}"
+
+    def make_message(self, message: str) -> str:
+        return f"({self.cut_id}) {message}"
+
+    def writer(self, message: str, is_err: bool = False) -> None:
+        """The path of logging data will store by format:
+
+            ... ./logs/run_id=<run-id>/stdout.txt
+            ... ./logs/run_id=<run-id>/stderr.txt
+
+        :param message:
+        :param is_err:
+        """
+        if not config.enable_write_log:
+            return
+
+        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
+        filename: str = frame_info.filename.split(os.path.sep)[-1]
+        lineno: int = frame_info.lineno
+
+        # NOTE: set process and thread IDs.
+        process: int = os.getpid()
+        thread: int = get_ident()
+
+        write_file: str = "stderr.txt" if is_err else "stdout.txt"
+        with (self.log_file / write_file).open(
+            mode="at", encoding="utf-8"
+        ) as f:
+            msg_fmt: str = f"{config.log_format_file}\n"
+            print(msg_fmt)
+            f.write(
+                msg_fmt.format(
+                    **{
+                        "datetime": get_dt_tznow().strftime(
+                            config.log_datetime_format
+                        ),
+                        "process": process,
+                        "thread": thread,
+                        "message": message,
+                        "filename": filename,
+                        "lineno": lineno,
+                    }
+                )
+            )
+
+
+@dataclass(
+    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+)
 class Result:
     """Result Pydantic Model for passing and receiving data context from any
     module execution process like stage execution, job execution, or workflow
@@ -28,22 +179,34 @@ class Result:
     and ``_run_id`` fields to comparing with other result instance.
     """
 
-    status: int = field(default=2)
+    status: Status = field(default=Status.WAIT)
     context: DictData = field(default_factory=dict)
-    run_id: Optional[str] = field(default=None)
+    run_id: Optional[str] = field(default_factory=default_gen_id)
 
     # NOTE: Ignore this field to compare another result model with __eq__.
     parent_run_id: Optional[str] = field(default=None, compare=False)
+    event: Event = field(default_factory=Event, compare=False)
+    ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    @model_validator(mode="after")
-    def __prepare_run_id(self) -> Self:
-        """Prepare running ID which use default ID if it initializes at the
-        first time.
-
-        :rtype: Self
+    @classmethod
+    def construct_with_rs_or_id(
+        cls,
+        result: Result | None = None,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        id_logic: str | None = None,
+    ) -> Self:  # pragma: no cov
+        """Create the Result object or set parent running id if passing Result
+        object.
         """
-        self._run_id = gen_id("manual", unique=True)
-        return self
+        if result is None:
+            result: Result = cls(
+                run_id=(run_id or gen_id(id_logic or "", unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
+        return result
 
     def set_run_id(self, running_id: str) -> Self:
         """Set a running ID.
@@ -51,7 +214,7 @@ class Result:
         :param running_id: A running ID that want to update on this model.
         :rtype: Self
         """
-        self.run_id = running_id
+        self.run_id: str = running_id
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
@@ -63,41 +226,31 @@ class Result:
         self.parent_run_id: str = running_id
         return self
 
-    def catch(self, status: int, context: DictData) -> Self:
-        """Catch the status and context to current data."""
-        self.__dict__["status"] = status
-        self.__dict__["context"].update(context)
-        return self
-
-    def receive(self, result: Result) -> Self:
-        """Receive context from another result object.
+    def catch(
+        self,
+        status: int | Status,
+        context: DictData | None = None,
+    ) -> Self:
+        """Catch the status and context to this Result object. This method will
+        use between a child execution return a result, and it wants to pass
+        status and context to this object.
 
-        :rtype: Self
+        :param status:
+        :param context:
         """
-        self.__dict__["status"] = result.status
-        self.__dict__["context"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id = result.parent_run_id
-        self.run_id = result.run_id
+        self.__dict__["status"] = (
+            Status(status) if isinstance(status, int) else status
+        )
+        self.__dict__["context"].update(context or {})
         return self
 
-    def receive_jobs(self, result: Result) -> Self:
-        """Receive context from another result object that use on the workflow
-        execution which create a ``jobs`` keys on the context if it does not
-        exist.
+    @property
+    def trace(self) -> TraceLog:
+        """Return TraceLog object that passing its running ID.
 
-        :rtype: Self
+        :rtype: TraceLog
         """
-        self.__dict__["status"] = result.status
-
-        # NOTE: Check the context has jobs key.
-        if "jobs" not in self.__dict__["context"]:
-            self.__dict__["context"]["jobs"] = {}
+        return TraceLog(self.run_id, self.parent_run_id)
 
-        self.__dict__["context"]["jobs"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id: str = result.parent_run_id
-        self.run_id: str = result.run_id
-        return self
+    def alive_time(self) -> float:  # pragma: no cov
+        return (get_dt_tznow() - self.ts).total_seconds()