ddeutil-workflow 0.0.36__py3-none-any.whl → 0.0.38__py3-none-any.whl

ddeutil/workflow/job.py

@@ -27,7 +27,7 @@ from threading import Event
 from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -43,7 +43,6 @@ from .stages import Stage
 from .templates import has_template
 from .utils import (
     cross_product,
-    dash2underscore,
     filter_func,
     gen_id,
 )
@@ -60,6 +59,8 @@ __all__: TupleStr = (
     "RunsOnSelfHosted",
     "RunsOnK8s",
     "make",
+    "local_execute_strategy",
+    "local_execute",
 )
 
 
@@ -153,7 +154,7 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
-        serialization_alias="fail-fast",
+        alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
@@ -162,7 +163,7 @@ class Strategy(BaseModel):
             "The maximum number of executor thread pool that want to run "
             "parallel"
         ),
-        serialization_alias="max-parallel",
+        alias="max-parallel",
     )
     matrix: Matrix = Field(
         default_factory=dict,
@@ -179,18 +180,6 @@ class Strategy(BaseModel):
         description="A list of exclude matrix that want to filter-out.",
     )
 
-    @model_validator(mode="before")
-    def __prepare_keys(cls, values: DictData) -> DictData:
-        """Rename key that use dash to underscore because Python does not
-        support this character exist in any variable name.
-
-        :param values: A parsing values to these models
-        :rtype: DictData
-        """
-        dash2underscore("max-parallel", values)
-        dash2underscore("fail-fast", values)
-        return values
-
     def is_set(self) -> bool:
         """Return True if this strategy was set from yaml template.
 
@@ -228,6 +217,10 @@ class RunsOnType(str, Enum):
 
 
 class BaseRunsOn(BaseModel):  # pragma: no cov
+    """Base Runs-On Model for generate runs-on types via inherit this model
+    object and override execute method.
+    """
+
     model_config = ConfigDict(use_enum_values=True)
 
     type: Literal[RunsOnType.LOCAL]
@@ -243,12 +236,17 @@ class RunsOnLocal(BaseRunsOn):  # pragma: no cov
     type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
 
 
+class SelfHostedArgs(BaseModel):
+    host: str
+
+
 class RunsOnSelfHosted(BaseRunsOn):  # pragma: no cov
     """Runs-on self-hosted."""
 
     type: Literal[RunsOnType.SELF_HOSTED] = Field(
         default=RunsOnType.SELF_HOSTED
     )
+    args: SelfHostedArgs = Field(alias="with")
 
 
 class RunsOnK8s(BaseRunsOn):  # pragma: no cov
@@ -257,13 +255,17 @@ class RunsOnK8s(BaseRunsOn):  # pragma: no cov
     type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
 
 
+def get_discriminator_runs_on(model: dict[str, Any]) -> str:
+    return model.get("type", "local")
+
+
 RunsOn = Annotated[
     Union[
-        RunsOnLocal,
-        RunsOnSelfHosted,
-        RunsOnK8s,
+        Annotated[RunsOnK8s, Tag(RunsOnType.K8S)],
+        Annotated[RunsOnSelfHosted, Tag(RunsOnType.SELF_HOSTED)],
+        Annotated[RunsOnLocal, Tag(RunsOnType.LOCAL)],
     ],
-    Field(discriminator="type"),
+    Discriminator(get_discriminator_runs_on),
 ]
 
 
@@ -308,7 +310,7 @@ class Job(BaseModel):
     runs_on: RunsOn = Field(
         default_factory=RunsOnLocal,
         description="A target node for this job to use for execution.",
-        serialization_alias="runs-on",
+        alias="runs-on",
     )
     stages: list[Stage] = Field(
         default_factory=list,
@@ -317,7 +319,7 @@ class Job(BaseModel):
     trigger_rule: TriggerRules = Field(
         default=TriggerRules.all_success,
         description="A trigger rule of tracking needed jobs.",
-        serialization_alias="trigger-rule",
+        alias="trigger-rule",
     )
     needs: list[str] = Field(
         default_factory=list,
@@ -328,18 +330,6 @@ class Job(BaseModel):
         description="A strategy matrix that want to generate.",
     )
 
-    @model_validator(mode="before")
-    def __prepare_keys__(cls, values: DictData) -> DictData:
-        """Rename key that use dash to underscore because Python does not
-        support this character exist in any variable name.
-
-        :param values: A passing value that coming for initialize this object.
-        :rtype: DictData
-        """
-        dash2underscore("runs-on", values)
-        dash2underscore("trigger-rule", values)
-        return values
-
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
         """Prepare description string that was created on a template.
@@ -430,15 +420,14 @@ class Job(BaseModel):
 
         :rtype: DictData
         """
+        if "jobs" not in to:
+            to["jobs"] = {}
+
         if self.id is None and not config.job_default_id:
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
 
-        # NOTE: Create jobs key to receive an output from the job execution.
-        if "jobs" not in to:
-            to["jobs"] = {}
-
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
@@ -447,278 +436,322 @@ class Job(BaseModel):
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
 
-        to["jobs"][_id] = (
-            {"strategies": output, **errors}
-            if self.strategy.is_set()
-            else {**output.get(next(iter(output), "DUMMY"), {}), **errors}
-        )
+        if self.strategy.is_set():
+            to["jobs"][_id] = {"strategies": output, **errors}
+        else:
+            _output = output.get(next(iter(output), "FIRST"), {})
+            _output.pop("matrix", {})
+            to["jobs"][_id] = {**_output, **errors}
         return to
 
-    def execute_strategy(
+    def execute(
         self,
-        strategy: DictData,
         params: DictData,
         *,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
         result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
-        """Job Strategy execution with passing dynamic parameters from the
-        workflow execution to strategy matrix.
-
-            This execution is the minimum level of execution of this job model.
-        It different with `self.execute` because this method run only one
-        strategy and return with context of this strategy data.
-
-            The result of this execution will return result with strategy ID
-        that generated from the `gen_id` function with an input strategy value.
-
-        :raise JobException: If it has any error from `StageException` or
-            `UtilException`.
+        """Job execution with passing dynamic parameters from the workflow
+        execution. It will generate matrix values at the first step and run
+        multithread on this metrics to the `stages` field of this job.
 
-        :param strategy: A strategy metrix value that use on this execution.
-            This value will pass to the `matrix` key for templating.
-        :param params: A dynamic parameters that will deepcopy to the context.
+        :param params: An input parameters that use on job execution.
+        :param run_id: A job running ID for this execution.
+        :param parent_run_id: A parent workflow running ID for this release.
         :param result: (Result) A result object for keeping context and status
             data.
-        :param event: An event manager that pass to the PoolThreadExecutor.
+        :param event: (Event) An event manager that pass to the
+            PoolThreadExecutor.
 
         :rtype: Result
         """
-        if result is None:  # pragma: no cov
-            result: Result = Result(run_id=gen_id(self.id or "", unique=True))
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=(self.id or "not-set"),
+        )
+
+        if self.runs_on.type == RunsOnType.LOCAL:
+            return local_execute(
+                job=self,
+                params=params,
+                result=result,
+                event=event,
+            )
+        elif self.runs_on.type == RunsOnType.SELF_HOSTED:  # pragma: no cov
+            pass
+        elif self.runs_on.type == RunsOnType.K8S:  # pragma: no cov
+            pass
+
+        # pragma: no cov
+        result.trace.error(
+            f"[JOB]: Job executor does not support for runs-on type: "
+            f"{self.runs_on.type} yet"
+        )
+        raise NotImplementedError(
+            f"The job runs-on other type: {self.runs_on.type} does not "
+            f"support yet."
+        )
+
+
+def local_execute_strategy(
+    job: Job,
+    strategy: DictData,
+    params: DictData,
+    *,
+    result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool = False,
+) -> Result:
+    """Local job strategy execution with passing dynamic parameters from the
+    workflow execution to strategy matrix.
+
+        This execution is the minimum level of execution of this job model.
+    It different with `self.execute` because this method run only one
+    strategy and return with context of this strategy data.
+
+        The result of this execution will return result with strategy ID
+    that generated from the `gen_id` function with an input strategy value.
+
+    :raise JobException: If it has any error from `StageException` or
+        `UtilException`.
+
+    :param job: (Job) A job model that want to execute.
+    :param strategy: A strategy metrix value that use on this execution.
+        This value will pass to the `matrix` key for templating.
+    :param params: A dynamic parameters that will deepcopy to the context.
+    :param result: (Result) A result object for keeping context and status
+        data.
+    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error
+
+    :rtype: Result
+    """
+    if result is None:
+        result: Result = Result(run_id=gen_id(job.id or "not-set", unique=True))
+
+    strategy_id: str = gen_id(strategy)
+
+    # PARAGRAPH:
+    #
+    #       Create strategy execution context and update a matrix and copied
+    #   of params. So, the context value will have structure like;
+    #
+    #   {
+    #       "params": { ... },      <== Current input params
+    #       "jobs": { ... },        <== Current input params
+    #       "matrix": { ... }       <== Current strategy value
+    #       "stages": { ... }       <== Catching stage outputs
+    #   }
+    #
+    context: DictData = copy.deepcopy(params)
+    context.update({"matrix": strategy, "stages": {}})
+
+    if strategy:
+        result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
+        result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
+
+    # IMPORTANT: The stage execution only run sequentially one-by-one.
+    for stage in job.stages:
+
+        if stage.is_skipped(params=context):
+            result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
+            continue
 
-        strategy_id: str = gen_id(strategy)
+        if event and event.is_set():
+            error_msg: str = (
+                "Job strategy was canceled from event that had set before "
+                "strategy execution."
+            )
+            return result.catch(
+                status=Status.FAILED,
+                context={
+                    strategy_id: {
+                        "matrix": strategy,
+                        "stages": context.pop("stages", {}),
+                        "errors": JobException(error_msg).to_dict(),
+                    },
+                },
+            )
 
         # PARAGRAPH:
         #
-        #       Create strategy execution context and update a matrix and copied
-        #   of params. So, the context value will have structure like;
+        #       This step will add the stage result to `stages` key in that
+        #   stage id. It will have structure like;
         #
         #   {
-        #       "params": { ... },      <== Current input params
-        #       "jobs": { ... },        <== Current input params
-        #       "matrix": { ... }       <== Current strategy value
-        #       "stages": { ... }       <== Catching stage outputs
+        #       "params": { ... },
+        #       "jobs": { ... },
+        #       "matrix": { ... },
+        #       "stages": { { "stage-id-01": { "outputs": { ... } } }, ... }
         #   }
         #
-        context: DictData = copy.deepcopy(params)
-        context.update({"matrix": strategy, "stages": {}})
-
-        # IMPORTANT: The stage execution only run sequentially one-by-one.
-        for stage in self.stages:
+        # IMPORTANT:
+        #   This execution change all stage running IDs to the current job
+        #   running ID, but it still trac log to the same parent running ID
+        #   (with passing `run_id` and `parent_run_id` to the stage
+        #   execution arguments).
+        #
+        try:
+            stage.set_outputs(
+                stage.handler_execute(
+                    params=context,
+                    run_id=result.run_id,
+                    parent_run_id=result.parent_run_id,
+                ).context,
+                to=context,
+            )
+        except (StageException, UtilException) as err:
+            result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
+            if raise_error or config.job_raise_error:
+                raise JobException(
+                    f"Stage execution error: {err.__class__.__name__}: "
+                    f"{err}"
+                ) from None
+
+            return result.catch(
+                status=Status.FAILED,
+                context={
+                    strategy_id: {
+                        "matrix": strategy,
+                        "stages": context.pop("stages", {}),
+                        "errors": err.to_dict(),
+                    },
+                },
+            )
 
-            if stage.is_skipped(params=context):
-                result.trace.info(f"[JOB]: Skip stage: {stage.iden!r}")
-                continue
+        # NOTE: Remove the current stage object for saving memory.
+        del stage
 
-            result.trace.info(f"[JOB]: Execute stage: {stage.iden!r}")
+    return result.catch(
+        status=Status.SUCCESS,
+        context={
+            strategy_id: {
+                "matrix": strategy,
+                "stages": filter_func(context.pop("stages", {})),
+            },
+        },
+    )
 
-            # NOTE: Logging a matrix that pass on this stage execution.
-            if strategy:
-                result.trace.info(f"[JOB]: ... Matrix: {strategy}")
 
-            # NOTE: Force stop this execution if event was set from main
-            #   execution.
-            if event and event.is_set():
-                error_msg: str = (
-                    "Job strategy was canceled from event that had set before "
-                    "strategy execution."
-                )
-                return result.catch(
-                    status=1,
-                    context={
-                        strategy_id: {
-                            "matrix": strategy,
-                            # NOTE: If job strategy executor use multithreading,
-                            #   it will not filter function object from context.
-                            # ---
-                            # "stages": filter_func(context.pop("stages", {})),
-                            #
-                            "stages": context.pop("stages", {}),
-                            "errors": {
-                                "class": JobException(error_msg),
-                                "name": "JobException",
-                                "message": error_msg,
-                            },
-                        },
-                    },
-                )
+def local_execute(
+    job: Job,
+    params: DictData,
+    *,
+    run_id: str | None = None,
+    parent_run_id: str | None = None,
+    result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool = False,
+) -> Result:
+    """Local job execution with passing dynamic parameters from the workflow
+    execution. It will generate matrix values at the first step and run
+    multithread on this metrics to the `stages` field of this job.
+
+        This method does not raise any JobException if it runs with
+    multi-threading strategy.
+
+    :param job: (Job) A job model that want to execute.
+    :param params: (DictData) An input parameters that use on job execution.
+    :param run_id: (str) A job running ID for this execution.
+    :param parent_run_id: (str) A parent workflow running ID for this release.
+    :param result: (Result) A result object for keeping context and status
+        data.
+    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error to the
+        strategy execution.
+
+    :rtype: Result
+    """
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+    )
+    event: Event = Event() if event is None else event
+
+    # NOTE: Normal Job execution without parallel strategy matrix. It uses
+    #   for-loop to control strategy execution sequentially.
+    if (not job.strategy.is_set()) or job.strategy.max_parallel == 1:
+
+        for strategy in job.strategy.make():
+
+            # TODO: stop and raise error if the event was set.
+            local_execute_strategy(
+                job=job,
+                strategy=strategy,
+                params=params,
+                result=result,
+                event=event,
+                raise_error=raise_error,
+            )
 
-            # PARAGRAPH:
-            #
-            #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer, and it was changed when I action
-            #   anything like update or re-construct this.
-            #
-            #       ... params |= stage.execute(params=params)
-            #
-            #   This step will add the stage result to `stages` key in
-            #   that stage id. It will have structure like;
-            #
-            #   {
-            #       "params": { ... },
-            #       "jobs": { ... },
-            #       "matrix": { ... },
-            #       "stages": { { "stage-id-1": ... }, ... }
-            #   }
-            #
-            # IMPORTANT:
-            #   This execution change all stage running IDs to the current job
-            #   running ID, but it still trac log to the same parent running ID
-            #   (with passing `run_id` and `parent_run_id` to the stage
-            #   execution arguments).
-            #
-            try:
-                stage.set_outputs(
-                    stage.handler_execute(
-                        params=context,
-                        run_id=result.run_id,
-                        parent_run_id=result.parent_run_id,
-                    ).context,
-                    to=context,
-                )
-            except (StageException, UtilException) as err:
-                result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-                if config.job_raise_error:
-                    raise JobException(
-                        f"Stage execution error: {err.__class__.__name__}: "
-                        f"{err}"
-                    ) from None
-
-                return result.catch(
-                    status=1,
-                    context={
-                        strategy_id: {
-                            "matrix": strategy,
-                            "stages": context.pop("stages", {}),
-                            "errors": {
-                                "class": err,
-                                "name": err.__class__.__name__,
-                                "message": f"{err.__class__.__name__}: {err}",
-                            },
-                        },
-                    },
-                )
+        return result.catch(status=Status.SUCCESS)
 
-            # NOTE: Remove the current stage object for saving memory.
-            del stage
+    fail_fast_flag: bool = job.strategy.fail_fast
+    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
 
-        return result.catch(
-            status=Status.SUCCESS,
-            context={
-                strategy_id: {
-                    "matrix": strategy,
-                    "stages": filter_func(context.pop("stages", {})),
-                },
-            },
-        )
+    result.trace.info(
+        f"[JOB]: Start multithreading: {job.strategy.max_parallel} threads "
+        f"with {ls} mode."
+    )
 
-    def execute(
-        self,
-        params: DictData,
-        *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-    ) -> Result:
-        """Job execution with passing dynamic parameters from the workflow
-        execution. It will generate matrix values at the first step and run
-        multithread on this metrics to the `stages` field of this job.
+    # IMPORTANT: Start running strategy execution by multithreading because
+    #   it will run by strategy values without waiting previous execution.
+    with ThreadPoolExecutor(
+        max_workers=job.strategy.max_parallel,
+        thread_name_prefix="job_strategy_exec_",
+    ) as executor:
+
+        futures: list[Future] = [
+            executor.submit(
+                local_execute_strategy,
+                job=job,
+                strategy=strategy,
+                params=params,
+                result=result,
+                event=event,
+                raise_error=raise_error,
+            )
+            for strategy in job.strategy.make()
+        ]
 
-        :param params: An input parameters that use on job execution.
-        :param run_id: A job running ID for this execution.
-        :param parent_run_id: A parent workflow running ID for this release.
-        :param result: (Result) A result object for keeping context and status
-            data.
+        context: DictData = {}
+        status: Status = Status.SUCCESS
 
-        :rtype: Result
-        """
-        if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(run_id or gen_id(self.id or "", unique=True)),
-                parent_run_id=parent_run_id,
+        if not fail_fast_flag:
+            done = as_completed(futures, timeout=1800)
+        else:
+            done, not_done = wait(
+                futures, timeout=1800, return_when=FIRST_EXCEPTION
             )
-        elif parent_run_id:  # pragma: no cov
-            result.set_parent_run_id(parent_run_id)
-
-        # NOTE: Normal Job execution without parallel strategy matrix. It uses
-        #   for-loop to control strategy execution sequentially.
-        if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
-
-            for strategy in self.strategy.make():
-                result: Result = self.execute_strategy(
-                    strategy=strategy,
-                    params=params,
-                    result=result,
-                )
 
-            return result.catch(status=Status.SUCCESS)
-
-        # NOTE: Create event for cancel executor by trigger stop running event.
-        event: Event = Event()
-
-        # IMPORTANT: Start running strategy execution by multithreading because
-        #   it will run by strategy values without waiting previous execution.
-        with ThreadPoolExecutor(
-            max_workers=self.strategy.max_parallel,
-            thread_name_prefix="job_strategy_exec_",
-        ) as executor:
-
-            futures: list[Future] = [
-                executor.submit(
-                    self.execute_strategy,
-                    strategy=strategy,
-                    params=params,
-                    result=result,
-                    event=event,
-                )
-                for strategy in self.strategy.make()
-            ]
-
-            context: DictData = {}
-            status: Status = Status.SUCCESS
-            fail_fast_flag: bool = self.strategy.fail_fast
-
-            if fail_fast_flag:
-                # NOTE: Get results from a collection of tasks with a timeout
-                #   that has the first exception.
-                done, not_done = wait(
-                    futures, timeout=1800, return_when=FIRST_EXCEPTION
+            if len(done) != len(futures):
+                result.trace.warning(
+                    "[JOB]: Set the event for stop running stage."
                 )
-                nd: str = (
-                    f", the strategies do not run is {not_done}"
-                    if not_done
-                    else ""
+                event.set()
+                for future in not_done:
+                    future.cancel()
+
+            nd: str = (
+                f", the strategies do not run is {not_done}" if not_done else ""
+            )
+            result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
+        for future in done:
+            try:
+                future.result()
+            except JobException as err:
+                status = Status.FAILED
+                result.trace.error(
+                    f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
+                    f"\n\t{err}"
                 )
-                result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
-
-                # NOTE: Stop all running tasks with setting the event manager
-                #   and cancel any scheduled tasks.
-                if len(done) != len(futures):
-                    event.set()
-                    for future in not_done:
-                        future.cancel()
-            else:
-                done = as_completed(futures, timeout=1800)
-
-            for future in done:
-                try:
-                    future.result()
-                except JobException as err:
-                    status = Status.FAILED
-                    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
-                    result.trace.error(
-                        f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
-                        f"\n\t{err}"
-                    )
-                    context.update(
-                        {
-                            "errors": {
-                                "class": err,
-                                "name": err.__class__.__name__,
-                                "message": f"{err.__class__.__name__}: {err}",
-                            },
-                        },
-                    )
-
-        return result.catch(status=status, context=context)
+                context.update({"errors": err.to_dict()})
+
+    return result.catch(status=status, context=context)