ddeutil-workflow 0.0.37__py3-none-any.whl → 0.0.39__py3-none-any.whl

ddeutil/workflow/job.py

@@ -27,7 +27,7 @@ from threading import Event
 from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -40,13 +40,8 @@ from .exceptions import (
 )
 from .result import Result, Status
 from .stages import Stage
-from .templates import has_template
-from .utils import (
-    cross_product,
-    dash2underscore,
-    filter_func,
-    gen_id,
-)
+from .templates import has_template, param2template
+from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -55,6 +50,7 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
+    "TriggerState",
     "RunsOn",
     "RunsOnLocal",
     "RunsOnSelfHosted",
@@ -155,7 +151,7 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
-        serialization_alias="fail-fast",
+        alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
@@ -164,7 +160,7 @@ class Strategy(BaseModel):
             "The maximum number of executor thread pool that want to run "
             "parallel"
         ),
-        serialization_alias="max-parallel",
+        alias="max-parallel",
     )
     matrix: Matrix = Field(
         default_factory=dict,
@@ -181,18 +177,6 @@ class Strategy(BaseModel):
         description="A list of exclude matrix that want to filter-out.",
     )
 
-    @model_validator(mode="before")
-    def __prepare_keys(cls, values: DictData) -> DictData:
-        """Rename key that use dash to underscore because Python does not
-        support this character exist in any variable name.
-
-        :param values: A parsing values to these models
-        :rtype: DictData
-        """
-        dash2underscore("max-parallel", values)
-        dash2underscore("fail-fast", values)
-        return values
-
     def is_set(self) -> bool:
         """Return True if this strategy was set from yaml template.
 
@@ -221,6 +205,16 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
+class TriggerState(str, Enum):
+    waiting: str = "waiting"
+    passed: str = "passed"
+    skipped: str = "skipped"
+    failed: str = "failed"
+
+    def is_waiting(self):
+        return self.value == "waiting"
+
+
 class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
@@ -268,13 +262,17 @@ class RunsOnK8s(BaseRunsOn):  # pragma: no cov
     type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
 
 
+def get_discriminator_runs_on(model: dict[str, Any]) -> str:
+    return model.get("type", "local")
+
+
 RunsOn = Annotated[
     Union[
-        RunsOnLocal,
-        RunsOnSelfHosted,
-        RunsOnK8s,
+        Annotated[RunsOnK8s, Tag(RunsOnType.K8S)],
+        Annotated[RunsOnSelfHosted, Tag(RunsOnType.SELF_HOSTED)],
+        Annotated[RunsOnLocal, Tag(RunsOnType.LOCAL)],
     ],
-    Field(discriminator="type"),
+    Discriminator(get_discriminator_runs_on),
 ]
 
 
@@ -319,7 +317,12 @@ class Job(BaseModel):
     runs_on: RunsOn = Field(
         default_factory=RunsOnLocal,
         description="A target node for this job to use for execution.",
-        serialization_alias="runs-on",
+        alias="runs-on",
+    )
+    condition: Optional[str] = Field(
+        default=None,
+        description="A job condition statement to allow job executable.",
+        alias="if",
     )
     stages: list[Stage] = Field(
         default_factory=list,
@@ -327,8 +330,11 @@ class Job(BaseModel):
     )
     trigger_rule: TriggerRules = Field(
         default=TriggerRules.all_success,
-        description="A trigger rule of tracking needed jobs.",
-        serialization_alias="trigger-rule",
+        description=(
+            "A trigger rule of tracking needed jobs if feature will use when "
+            "the `raise_error` did not set from job and stage executions."
+        ),
+        alias="trigger-rule",
     )
     needs: list[str] = Field(
         default_factory=list,
@@ -339,18 +345,6 @@ class Job(BaseModel):
         description="A strategy matrix that want to generate.",
     )
 
-    @model_validator(mode="before")
-    def __prepare_keys__(cls, values: DictData) -> DictData:
-        """Rename key that use dash to underscore because Python does not
-        support this character exist in any variable name.
-
-        :param values: A passing value that coming for initialize this object.
-        :rtype: DictData
-        """
-        dash2underscore("runs-on", values)
-        dash2underscore("trigger-rule", values)
-        return values
-
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
         """Prepare description string that was created on a template.
@@ -403,12 +397,87 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
-    def check_needs(self, jobs: dict[str, Any]) -> bool:
+    def check_needs(
+        self, jobs: dict[str, Any]
+    ) -> TriggerState:  # pragma: no cov
         """Return True if job's need exists in an input list of job's ID.
 
+        :param jobs: A mapping of job model and its ID.
+
+        :rtype: TriggerState
+        """
+        if not self.needs:
+            return TriggerState.passed
+
+        def make_return(result: bool) -> TriggerState:
+            return TriggerState.passed if result else TriggerState.failed
+
+        need_exist: dict[str, Any] = {
+            need: jobs[need] for need in self.needs if need in jobs
+        }
+        if len(need_exist) != len(self.needs):
+            return TriggerState.waiting
+        elif all("skipped" in need_exist[job] for job in need_exist):
+            return TriggerState.skipped
+        elif self.trigger_rule == TriggerRules.all_done:
+            return TriggerState.passed
+        elif self.trigger_rule == TriggerRules.all_success:
+            rs = all(
+                k not in need_exist[job]
+                for k in ("errors", "skipped")
+                for job in need_exist
+            )
+        elif self.trigger_rule == TriggerRules.all_failed:
+            rs = all("errors" in need_exist[job] for job in need_exist)
+        elif self.trigger_rule == TriggerRules.one_success:
+            rs = sum(
+                k not in need_exist[job]
+                for k in ("errors", "skipped")
+                for job in need_exist
+            ) + 1 == len(self.needs)
+        elif self.trigger_rule == TriggerRules.one_failed:
+            rs = sum("errors" in need_exist[job] for job in need_exist) == 1
+        elif self.trigger_rule == TriggerRules.none_skipped:
+            rs = all("skipped" not in need_exist[job] for job in need_exist)
+        elif self.trigger_rule == TriggerRules.none_failed:
+            rs = all("errors" not in need_exist[job] for job in need_exist)
+        else:  # pragma: no cov
+            raise NotImplementedError(
+                f"Trigger rule: {self.trigger_rule} does not support yet."
+            )
+        return make_return(rs)
+
+    def is_skipped(self, params: DictData | None = None) -> bool:
+        """Return true if condition of this job do not correct. This process
+        use build-in eval function to execute the if-condition.
+
+        :raise JobException: When it has any error raise from the eval
+            condition statement.
+        :raise JobException: When return type of the eval condition statement
+            does not return with boolean type.
+
+        :param params: (DictData) A parameters that want to pass to condition
+            template.
+
         :rtype: bool
         """
-        return all(need in jobs for need in self.needs)
+        if self.condition is None:
+            return False
+
+        params: DictData = {} if params is None else params
+
+        try:
+            # WARNING: The eval build-in function is very dangerous. So, it
+            #   should use the `re` module to validate eval-string before
+            #   running.
+            rs: bool = eval(
+                param2template(self.condition, params), globals() | params, {}
+            )
+            if not isinstance(rs, bool):
+                raise TypeError("Return type of condition does not be boolean")
+            return not rs
+        except Exception as err:
+            raise JobException(f"{err.__class__.__name__}: {err}") from err
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
@@ -441,15 +510,14 @@ class Job(BaseModel):
 
         :rtype: DictData
         """
+        if "jobs" not in to:
+            to["jobs"] = {}
+
         if self.id is None and not config.job_default_id:
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
 
-        # NOTE: Create jobs key to receive an output from the job execution.
-        if "jobs" not in to:
-            to["jobs"] = {}
-
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
@@ -458,11 +526,14 @@ class Job(BaseModel):
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
 
-        to["jobs"][_id] = (
-            {"strategies": output, **errors}
-            if self.strategy.is_set()
-            else {**output.get(next(iter(output), "DUMMY"), {}), **errors}
-        )
+        if "SKIP" in output:  # pragma: no cov
+            to["jobs"][_id] = output["SKIP"]
+        elif self.strategy.is_set():
+            to["jobs"][_id] = {"strategies": output, **errors}
+        else:
+            _output = output.get(next(iter(output), "FIRST"), {})
+            _output.pop("matrix", {})
+            to["jobs"][_id] = {**_output, **errors}
         return to
 
     def execute(
@@ -472,33 +543,46 @@ class Job(BaseModel):
         run_id: str | None = None,
         parent_run_id: str | None = None,
         result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the `stages` field of this job.
 
         :param params: An input parameters that use on job execution.
-        :param run_id: A job running ID for this execution.
-        :param parent_run_id: A parent workflow running ID for this release.
+        :param run_id: (str) A job running ID.
+        :param parent_run_id: (str) A parent workflow running ID.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that pass to the
+            PoolThreadExecutor.
 
         :rtype: Result
         """
-        if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(run_id or gen_id(self.id or "", unique=True)),
-                parent_run_id=parent_run_id,
-            )
-        elif parent_run_id:  # pragma: no cov
-            result.set_parent_run_id(parent_run_id)
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=(self.id or "not-set"),
+        )
 
         if self.runs_on.type == RunsOnType.LOCAL:
             return local_execute(
                 job=self,
                 params=params,
                 result=result,
+                event=event,
             )
+        elif self.runs_on.type == RunsOnType.SELF_HOSTED:  # pragma: no cov
+            pass
+        elif self.runs_on.type == RunsOnType.K8S:  # pragma: no cov
+            pass
+
+        # pragma: no cov
+        result.trace.error(
+            f"[JOB]: Job executor does not support for runs-on type: "
+            f"{self.runs_on.type} yet"
+        )
         raise NotImplementedError(
             f"The job runs-on other type: {self.runs_on.type} does not "
             f"support yet."
@@ -512,6 +596,7 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
+    raise_error: bool = False,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -533,11 +618,12 @@ def local_execute_strategy(
     :param result: (Result) A result object for keeping context and status
         data.
     :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error
 
     :rtype: Result
     """
-    if result is None:  # pragma: no cov
-        result: Result = Result(run_id=gen_id(job.id or "", unique=True))
+    if result is None:
+        result: Result = Result(run_id=gen_id(job.id or "not-set", unique=True))
 
     strategy_id: str = gen_id(strategy)
 
@@ -556,62 +642,44 @@ def local_execute_strategy(
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
 
+    if strategy:
+        result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
+        result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
+
     # IMPORTANT: The stage execution only run sequentially one-by-one.
     for stage in job.stages:
 
         if stage.is_skipped(params=context):
-            result.trace.info(f"[JOB]: Skip stage: {stage.iden!r}")
+            result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
+            stage.set_outputs(output={"skipped": True}, to=context)
             continue
 
-        result.trace.info(f"[JOB]: Execute stage: {stage.iden!r}")
-
-        # NOTE: Logging a matrix that pass on this stage execution.
-        if strategy:
-            result.trace.info(f"[JOB]: ... Matrix: {strategy}")
-
-        # NOTE: Force stop this execution if event was set from main
-        #   execution.
         if event and event.is_set():
             error_msg: str = (
                 "Job strategy was canceled from event that had set before "
                 "strategy execution."
             )
             return result.catch(
-                status=1,
+                status=Status.FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
-                        # NOTE: If job strategy executor use multithreading,
-                        #   it will not filter function object from context.
-                        # ---
-                        # "stages": filter_func(context.pop("stages", {})),
-                        #
                         "stages": context.pop("stages", {}),
-                        "errors": {
-                            "class": JobException(error_msg),
-                            "name": "JobException",
-                            "message": error_msg,
-                        },
+                        "errors": JobException(error_msg).to_dict(),
                     },
                 },
             )
 
         # PARAGRAPH:
         #
-        #       I do not use below syntax because `params` dict be the
-        #   reference memory pointer, and it was changed when I action
-        #   anything like update or re-construct this.
-        #
-        #       ... params |= stage.execute(params=params)
-        #
-        #   This step will add the stage result to `stages` key in
-        #   that stage id. It will have structure like;
+        #       This step will add the stage result to `stages` key in that
+        #   stage id. It will have structure like;
         #
         #   {
         #       "params": { ... },
         #       "jobs": { ... },
         #       "matrix": { ... },
-        #       "stages": { { "stage-id-1": ... }, ... }
+        #       "stages": { { "stage-id-01": { "outputs": { ... } } }, ... }
         #   }
         #
         # IMPORTANT:
@@ -631,30 +699,23 @@ def local_execute_strategy(
             )
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            if config.job_raise_error:
+            if raise_error or config.job_raise_error:
                 raise JobException(
                     f"Stage execution error: {err.__class__.__name__}: "
                     f"{err}"
                 ) from None
 
             return result.catch(
-                status=1,
+                status=Status.FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
                         "stages": context.pop("stages", {}),
-                        "errors": {
-                            "class": err,
-                            "name": err.__class__.__name__,
-                            "message": f"{err.__class__.__name__}: {err}",
-                        },
+                        "errors": err.to_dict(),
                     },
                 },
             )
 
-        # NOTE: Remove the current stage object for saving memory.
-        del stage
-
     return result.catch(
         status=Status.SUCCESS,
         context={
@@ -673,44 +734,81 @@ def local_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool = False,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution. It will generate matrix values at the first step and run
     multithread on this metrics to the `stages` field of this job.
 
-    :param job: A job model that want to execute.
-    :param params: An input parameters that use on job execution.
-    :param run_id: A job running ID for this execution.
-    :param parent_run_id: A parent workflow running ID for this release.
+        This method does not raise any JobException if it runs with
+    multi-threading strategy.
+
+    :param job: (Job) A job model that want to execute.
+    :param params: (DictData) An input parameters that use on job execution.
+    :param run_id: (str) A job running ID for this execution.
+    :param parent_run_id: (str) A parent workflow running ID for this release.
     :param result: (Result) A result object for keeping context and status
         data.
+    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error to the
+        strategy execution.
 
     :rtype: Result
     """
-    if result is None:  # pragma: no cov
-        result: Result = Result(
-            run_id=(run_id or gen_id(job.id or "", unique=True)),
-            parent_run_id=parent_run_id,
-        )
-    elif parent_run_id:  # pragma: no cov
-        result.set_parent_run_id(parent_run_id)
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+    )
+    event: Event = Event() if event is None else event
 
     # NOTE: Normal Job execution without parallel strategy matrix. It uses
     #   for-loop to control strategy execution sequentially.
     if (not job.strategy.is_set()) or job.strategy.max_parallel == 1:
 
         for strategy in job.strategy.make():
-            result: Result = local_execute_strategy(
+
+            if event and event.is_set():  # pragma: no cov
+                return result.catch(
+                    status=Status.FAILED,
+                    context={
+                        "errors": JobException(
+                            "Job strategy was canceled from event that had set "
+                            "before strategy execution."
+                        ).to_dict()
+                    },
+                )
+
+            local_execute_strategy(
                 job=job,
                 strategy=strategy,
                 params=params,
                 result=result,
+                event=event,
+                raise_error=raise_error,
             )
 
         return result.catch(status=Status.SUCCESS)
 
-    # NOTE: Create event for cancel executor by trigger stop running event.
-    event: Event = Event()
+    fail_fast_flag: bool = job.strategy.fail_fast
+    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
+    result.trace.info(
+        f"[JOB]: Start multithreading: {job.strategy.max_parallel} threads "
+        f"with {ls} mode."
+    )
+
+    if event and event.is_set():  # pragma: no cov
+        return result.catch(
+            status=Status.FAILED,
+            context={
+                "errors": JobException(
+                    "Job strategy was canceled from event that had set "
+                    "before strategy execution."
+                ).to_dict()
+            },
+        )
 
     # IMPORTANT: Start running strategy execution by multithreading because
     #   it will run by strategy values without waiting previous execution.
@@ -727,52 +825,43 @@ def local_execute(
                 params=params,
                 result=result,
                 event=event,
+                raise_error=raise_error,
             )
             for strategy in job.strategy.make()
         ]
 
         context: DictData = {}
         status: Status = Status.SUCCESS
-        fail_fast_flag: bool = job.strategy.fail_fast
 
         if not fail_fast_flag:
             done = as_completed(futures, timeout=1800)
         else:
-            # NOTE: Get results from a collection of tasks with a timeout
-            #   that has the first exception.
             done, not_done = wait(
                 futures, timeout=1800, return_when=FIRST_EXCEPTION
             )
-            nd: str = (
-                f", the strategies do not run is {not_done}" if not_done else ""
-            )
-            result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
 
-            # NOTE: Stop all running tasks with setting the event manager
-            #   and cancel any scheduled tasks.
             if len(done) != len(futures):
+                result.trace.warning(
+                    "[JOB]: Set the event for stop running stage."
+                )
                 event.set()
                 for future in not_done:
                     future.cancel()
 
+            nd: str = (
+                f", the strategies do not run is {not_done}" if not_done else ""
+            )
+            result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
         for future in done:
             try:
                 future.result()
             except JobException as err:
                 status = Status.FAILED
-                ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
                 result.trace.error(
                     f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
                     f"\n\t{err}"
                 )
-                context.update(
-                    {
-                        "errors": {
-                            "class": err,
-                            "name": err.__class__.__name__,
-                            "message": f"{err.__class__.__name__}: {err}",
-                        },
-                    },
-                )
+                context.update({"errors": err.to_dict()})
 
     return result.catch(status=status, context=context)

ddeutil/workflow/logs.py

@@ -234,7 +234,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         return f"({self.cut_id}) {message}"
 
     def writer(self, message: str, is_err: bool = False) -> None:
-        """ "Write a trace message after making to target file and write metadata
+        """Write a trace message after making to target file and write metadata
         in the same path of standard files.
 
             The path of logging data will store by format:
@@ -279,6 +279,11 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
                 + "\n"
             )
 
+    async def awriter(
+        self, message: str, is_err: bool = False
+    ):  # pragma: no cov
+        """TODO: Use `aiofiles` for make writer method support async."""
+
 
 class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
     """Trace Log object that write trace log to the SQLite database file."""

ddeutil/workflow/result.py

@@ -91,7 +91,7 @@ class Result:
     @model_validator(mode="after")
     def __prepare_trace(self) -> Self:
         """Prepare trace field that want to pass after its initialize step."""
-        if self.trace is None:  # pragma: no cove
+        if self.trace is None:  # pragma: no cov
             self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         return self