ddeutil-workflow 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

ddeutil/workflow/job.py

@@ -38,12 +38,11 @@ from .exceptions import (
     StageException,
     UtilException,
 )
-from .result import Result
+from .result import Result, Status
 from .stage import Stage
 from .templates import has_template
 from .utils import (
     cross_product,
-    cut_id,
     dash2underscore,
     filter_func,
     gen_id,
@@ -222,6 +221,8 @@ class RunsOn(str, Enum):
 
     local: str = "local"
     docker: str = "docker"
+    self_hosted: str = "self_hosted"
+    k8s: str = "k8s"
 
 
 class Job(BaseModel):
@@ -412,6 +413,7 @@ class Job(BaseModel):
         params: DictData,
         *,
         run_id: str | None = None,
+        result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
@@ -431,13 +433,18 @@ class Job(BaseModel):
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :param event: An event manager that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
-        run_id: str = run_id or gen_id(self.id or "", unique=True)
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=(run_id or gen_id(self.id or "", unique=True))
+            )
+
         strategy_id: str = gen_id(strategy)
-        rs: Result = Result(run_id=run_id)
 
         # PARAGRAPH:
         #
@@ -458,18 +465,14 @@ class Job(BaseModel):
         for stage in self.stages:
 
             if stage.is_skipped(params=context):
-                logger.info(
-                    f"({cut_id(run_id)}) [JOB]: Skip stage: {stage.iden!r}"
-                )
+                result.trace.info(f"[JOB]: Skip stage: {stage.iden!r}")
                 continue
 
-            logger.info(
-                f"({cut_id(run_id)}) [JOB]: Execute stage: {stage.iden!r}"
-            )
+            result.trace.info(f"[JOB]: Execute stage: {stage.iden!r}")
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({cut_id(run_id)}) [JOB]: ... Matrix: {strategy}")
+                result.trace.info(f"[JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -478,7 +481,7 @@ class Job(BaseModel):
                     "Job strategy was canceled from event that had set before "
                     "strategy execution."
                 )
-                return rs.catch(
+                return result.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -516,20 +519,18 @@ class Job(BaseModel):
             try:
                 stage.set_outputs(
                     stage.handler_execute(
-                        params=context, run_id=run_id
+                        params=context, run_id=result.run_id
                     ).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
-                logger.error(
-                    f"({cut_id(run_id)}) [JOB]: {err.__class__.__name__}: {err}"
-                )
+                result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
                 if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                return rs.catch(
+                return result.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -544,8 +545,8 @@ class Job(BaseModel):
             # NOTE: Remove the current stage object for saving memory.
             del stage
 
-        return rs.catch(
-            status=0,
+        return result.catch(
+            status=Status.SUCCESS,
             context={
                 strategy_id: {
                     "matrix": strategy,
@@ -554,36 +555,43 @@ class Job(BaseModel):
             },
         )
 
-    def execute(self, params: DictData, run_id: str | None = None) -> Result:
+    def execute(
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        result: Result | None = None,
+    ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the ``stages`` field of this job.
 
         :param params: An input parameters that use on job execution.
         :param run_id: A job running ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
 
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
-        run_id: str = run_id or gen_id(self.id or "", unique=True)
-        context: DictData = {}
+        if result is None:  # pragma: no cov
+            run_id: str = run_id or gen_id(self.id or "", unique=True)
+            result: Result = Result(run_id=run_id)
 
         # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
+
             for strategy in self.strategy.make():
-                rs: Result = self.execute_strategy(
+                result: Result = self.execute_strategy(
                     strategy=strategy,
                     params=params,
-                    run_id=run_id,
+                    result=result,
                 )
-                context.update(rs.context)
-            return Result(
-                status=0,
-                context=context,
-            )
+
+            return result.catch(status=Status.SUCCESS)
 
         # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
@@ -600,23 +608,23 @@ class Job(BaseModel):
                     self.execute_strategy,
                     strategy=strategy,
                     params=params,
-                    run_id=run_id,
+                    result=result,
                     event=event,
                 )
                 for strategy in self.strategy.make()
             ]
 
             return (
-                self.__catch_fail_fast(event, futures=futures, run_id=run_id)
+                self.__catch_fail_fast(event, futures=futures, result=result)
                 if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures, run_id=run_id)
+                else self.__catch_all_completed(futures=futures, result=result)
             )
 
     @staticmethod
     def __catch_fail_fast(
         event: Event,
         futures: list[Future],
-        run_id: str,
+        result: Result,
         *,
         timeout: int = 1800,
     ) -> Result:
@@ -627,14 +635,14 @@ class Job(BaseModel):
         :param event: An event manager instance that able to set stopper on the
             observing multithreading.
         :param futures: A list of futures.
-        :param run_id: A job running ID from execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :param timeout: A timeout to waiting all futures complete.
 
         :rtype: Result
         """
-        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
-        status: int = 0
+        status: Status = Status.SUCCESS
 
         # NOTE: Get results from a collection of tasks with a timeout that has
         #   the first exception.
@@ -644,7 +652,7 @@ class Job(BaseModel):
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({cut_id(run_id)}) [JOB]: Strategy is set Fail Fast{nd}")
+        result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -660,10 +668,9 @@ class Job(BaseModel):
 
             # NOTE: Handle the first exception from feature
             if err := future.exception():
-                status: int = 1
-                logger.error(
-                    f"({cut_id(run_id)}) [JOB]: Fail-fast catching:\n\t"
-                    f"{future.exception()}"
+                status: Status = Status.FAILED
+                result.trace.error(
+                    f"[JOB]: Fail-fast catching:\n\t{future.exception()}"
                 )
                 context.update(
                     {
@@ -674,36 +681,36 @@ class Job(BaseModel):
                 continue
 
             # NOTE: Update the result context to main job context.
-            context.update(future.result().context)
+            future.result()
 
-        return rs_final.catch(status=status, context=context)
+        return result.catch(status=status, context=context)
 
     @staticmethod
     def __catch_all_completed(
         futures: list[Future],
-        run_id: str,
+        result: Result,
         *,
         timeout: int = 1800,
     ) -> Result:
         """Job parallel pool futures catching with all-completed mode.
 
         :param futures: A list of futures.
-        :param run_id: A job running ID from execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :param timeout: A timeout to waiting all futures complete.
 
         :rtype: Result
         """
-        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
-        status: int = 0
+        status: Status = Status.SUCCESS
 
         for future in as_completed(futures, timeout=timeout):
             try:
-                context.update(future.result().context)
+                future.result()
             except JobException as err:
-                status = 1
-                logger.error(
-                    f"({cut_id(run_id)}) [JOB]: All-completed catching:\n\t"
+                status = Status.FAILED
+                result.trace.error(
+                    f"[JOB]: All-completed catching:\n\t"
                     f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(
@@ -713,4 +720,4 @@ class Job(BaseModel):
                     },
                 )
 
-        return rs_final.catch(status=status, context=context)
+        return result.catch(status=status, context=context)

ddeutil/workflow/result.py

@@ -6,19 +6,76 @@
 from __future__ import annotations
 
 from dataclasses import field
+from datetime import datetime
+from enum import IntEnum
+from threading import Event
 from typing import Optional
 
+from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
-from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .utils import gen_id
+from .conf import config, get_logger
+from .utils import cut_id, gen_id, get_dt_now
 
-__all__: TupleStr = ("Result",)
+logger = get_logger("ddeutil.workflow.audit")
 
+__all__: TupleStr = (
+    "Result",
+    "Status",
+)
 
-@dataclass
+
+def default_gen_id() -> str:
+    """Return running ID which use for making default ID for the Result model if
+    a run_id field initializes at the first time.
+
+    :rtype: str
+    """
+    return gen_id("manual", unique=True)
+
+
+def get_dt_tznow() -> datetime:
+    """Return the current datetime object that passing the config timezone.
+
+    :rtype: datetime
+    """
+    return get_dt_now(tz=config.tz)
+
+
+class Status(IntEnum):
+    """Status Int Enum object."""
+
+    SUCCESS: int = 0
+    FAILED: int = 1
+    WAIT: int = 2
+
+
+class TraceLog:  # pragma: no cov
+    """Trace Log object."""
+
+    __slots__: TupleStr = ("run_id",)
+
+    def __init__(self, run_id: str):
+        self.run_id: str = run_id
+
+    def debug(self, message: str):
+        logger.debug(f"({cut_id(self.run_id)}) {message}")
+
+    def info(self, message: str):
+        logger.info(f"({cut_id(self.run_id)}) {message}")
+
+    def warning(self, message: str):
+        logger.warning(f"({cut_id(self.run_id)}) {message}")
+
+    def error(self, message: str):
+        logger.error(f"({cut_id(self.run_id)}) {message}")
+
+
+@dataclass(
+    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+)
 class Result:
     """Result Pydantic Model for passing and receiving data context from any
     module execution process like stage execution, job execution, or workflow
@@ -28,22 +85,14 @@ class Result:
     and ``_run_id`` fields to comparing with other result instance.
     """
 
-    status: int = field(default=2)
+    status: Status = field(default=Status.WAIT)
     context: DictData = field(default_factory=dict)
-    run_id: Optional[str] = field(default=None)
+    run_id: Optional[str] = field(default_factory=default_gen_id)
 
     # NOTE: Ignore this field to compare another result model with __eq__.
     parent_run_id: Optional[str] = field(default=None, compare=False)
-
-    @model_validator(mode="after")
-    def __prepare_run_id(self) -> Self:
-        """Prepare running ID which use default ID if it initializes at the
-        first time.
-
-        :rtype: Self
-        """
-        self._run_id = gen_id("manual", unique=True)
-        return self
+    event: Event = field(default_factory=Event, compare=False)
+    ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
     def set_run_id(self, running_id: str) -> Self:
         """Set a running ID.
@@ -51,7 +100,7 @@ class Result:
         :param running_id: A running ID that want to update on this model.
         :rtype: Self
         """
-        self.run_id = running_id
+        self.run_id: str = running_id
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
@@ -63,10 +112,22 @@ class Result:
         self.parent_run_id: str = running_id
         return self
 
-    def catch(self, status: int, context: DictData) -> Self:
-        """Catch the status and context to current data."""
-        self.__dict__["status"] = status
-        self.__dict__["context"].update(context)
+    def catch(
+        self,
+        status: int | Status,
+        context: DictData | None = None,
+    ) -> Self:
+        """Catch the status and context to this Result object. This method will
+        use between a child execution return a result, and it wants to pass
+        status and context to this object.
+
+        :param status:
+        :param context:
+        """
+        self.__dict__["status"] = (
+            Status(status) if isinstance(status, int) else status
+        )
+        self.__dict__["context"].update(context or {})
         return self
 
     def receive(self, result: Result) -> Self:
@@ -82,22 +143,13 @@ class Result:
         self.run_id = result.run_id
         return self
 
-    def receive_jobs(self, result: Result) -> Self:
-        """Receive context from another result object that use on the workflow
-        execution which create a ``jobs`` keys on the context if it does not
-        exist.
+    @property
+    def trace(self) -> TraceLog:
+        """Return TraceLog object that passing its running ID.
 
-        :rtype: Self
+        :rtype: TraceLog
         """
-        self.__dict__["status"] = result.status
+        return TraceLog(self.run_id)
 
-        # NOTE: Check the context has jobs key.
-        if "jobs" not in self.__dict__["context"]:
-            self.__dict__["context"]["jobs"] = {}
-
-        self.__dict__["context"]["jobs"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id: str = result.parent_run_id
-        self.run_id: str = result.run_id
-        return self
+    def alive_time(self) -> float:  # pragma: no cov
+        return (get_dt_tznow() - self.ts).total_seconds()

ddeutil/workflow/scheduler.py

@@ -51,7 +51,8 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, Log, config, get_log, get_logger
+from .audit import Audit, get_audit
+from .conf import Loader, config, get_logger
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
 from .result import Result
@@ -313,7 +314,7 @@ class Schedule(BaseModel):
         *,
         stop: datetime | None = None,
         externals: DictData | None = None,
-        log: type[Log] | None = None,
+        log: type[Audit] | None = None,
     ) -> None:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
@@ -330,7 +331,7 @@ class Schedule(BaseModel):
             ) from None
 
         # NOTE: Get default logging.
-        log: type[Log] = log or get_log()
+        log: type[Audit] = log or get_audit()
         scheduler: Scheduler = Scheduler()
 
         # NOTE: Create the start and stop datetime.
@@ -450,7 +451,7 @@ def schedule_task(
     stop: datetime,
     queue: dict[str, ReleaseQueue],
     threads: ReleaseThreads,
-    log: type[Log],
+    log: type[Audit],
 ) -> type[CancelJob] | None:
     """Schedule task function that generate thread of workflow task release
     method in background. This function do the same logic as the workflow poke
@@ -573,7 +574,7 @@ def schedule_control(
     stop: datetime | None = None,
     externals: DictData | None = None,
     *,
-    log: type[Log] | None = None,
+    log: type[Audit] | None = None,
 ) -> list[str]:  # pragma: no cov
     """Scheduler control function that run the chuck of schedules every minute
     and this function release monitoring thread for tracking undead thread in
@@ -596,7 +597,7 @@ def schedule_control(
         ) from None
 
     # NOTE: Get default logging.
-    log: type[Log] = log or get_log()
+    log: type[Audit] = log or get_audit()
     scheduler: Scheduler = Scheduler()
 
     # NOTE: Create the start and stop datetime.