ddeutil-workflow 0.0.48__py3-none-any.whl → 0.0.50__py3-none-any.whl

ddeutil/workflow/job.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
 mean the job model able to define `runs-on` key that allow you to run this
@@ -33,7 +32,6 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import dynamic
 from .exceptions import (
     JobException,
     StageException,
@@ -51,11 +49,12 @@ MatrixFilter = list[dict[str, Union[str, int]]]
 __all__: TupleStr = (
     "Strategy",
     "Job",
-    "TriggerRules",
+    "Rule",
     "RunsOn",
-    "RunsOnLocal",
-    "RunsOnSelfHosted",
-    "RunsOnK8s",
+    "RunsOnModel",
+    "OnLocal",
+    "OnSelfHosted",
+    "OnK8s",
     "make",
     "local_execute_strategy",
     "local_execute",
@@ -194,24 +193,25 @@ class Strategy(BaseModel):
         return make(self.matrix, self.include, self.exclude)
 
 
-class TriggerRules(str, Enum):
+class Rule(str, Enum):
     """Trigger rules enum object."""
 
-    all_success: str = "all_success"
-    all_failed: str = "all_failed"
-    all_done: str = "all_done"
-    one_failed: str = "one_failed"
-    one_success: str = "one_success"
-    none_failed: str = "none_failed"
-    none_skipped: str = "none_skipped"
+    ALL_SUCCESS: str = "all_success"
+    ALL_FAILED: str = "all_failed"
+    ALL_DONE: str = "all_done"
+    ONE_FAILED: str = "one_failed"
+    ONE_SUCCESS: str = "one_success"
+    NONE_FAILED: str = "none_failed"
+    NONE_SKIPPED: str = "none_skipped"
 
 
-class RunsOnType(str, Enum):
+class RunsOn(str, Enum):
     """Runs-On enum object."""
 
     LOCAL: str = "local"
     SELF_HOSTED: str = "self_hosted"
     K8S: str = "k8s"
+    AZ_BATCH: str = "azure_batch"
 
 
 class BaseRunsOn(BaseModel):  # pragma: no cov
@@ -221,47 +221,45 @@ class BaseRunsOn(BaseModel):  # pragma: no cov
 
     model_config = ConfigDict(use_enum_values=True)
 
-    type: Literal[RunsOnType.LOCAL]
+    type: Literal[RunsOn.LOCAL]
     args: DictData = Field(
         default_factory=dict,
         alias="with",
     )
 
 
-class RunsOnLocal(BaseRunsOn):  # pragma: no cov
+class OnLocal(BaseRunsOn):  # pragma: no cov
     """Runs-on local."""
 
-    type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
+    type: Literal[RunsOn.LOCAL] = Field(default=RunsOn.LOCAL)
 
 
 class SelfHostedArgs(BaseModel):
     host: str
 
 
-class RunsOnSelfHosted(BaseRunsOn):  # pragma: no cov
+class OnSelfHosted(BaseRunsOn):  # pragma: no cov
     """Runs-on self-hosted."""
 
-    type: Literal[RunsOnType.SELF_HOSTED] = Field(
-        default=RunsOnType.SELF_HOSTED
-    )
+    type: Literal[RunsOn.SELF_HOSTED] = Field(default=RunsOn.SELF_HOSTED)
     args: SelfHostedArgs = Field(alias="with")
 
 
-class RunsOnK8s(BaseRunsOn):  # pragma: no cov
+class OnK8s(BaseRunsOn):  # pragma: no cov
     """Runs-on Kubernetes."""
 
-    type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
+    type: Literal[RunsOn.K8S] = Field(default=RunsOn.K8S)
 
 
 def get_discriminator_runs_on(model: dict[str, Any]) -> str:
     return model.get("type", "local")
 
 
-RunsOn = Annotated[
+RunsOnModel = Annotated[
     Union[
-        Annotated[RunsOnK8s, Tag(RunsOnType.K8S)],
-        Annotated[RunsOnSelfHosted, Tag(RunsOnType.SELF_HOSTED)],
-        Annotated[RunsOnLocal, Tag(RunsOnType.LOCAL)],
+        Annotated[OnK8s, Tag(RunsOn.K8S)],
+        Annotated[OnSelfHosted, Tag(RunsOn.SELF_HOSTED)],
+        Annotated[OnLocal, Tag(RunsOn.LOCAL)],
     ],
     Discriminator(get_discriminator_runs_on),
 ]
@@ -305,8 +303,8 @@ class Job(BaseModel):
         default=None,
         description="A job description that can be string of markdown content.",
     )
-    runs_on: RunsOn = Field(
-        default_factory=RunsOnLocal,
+    runs_on: RunsOnModel = Field(
+        default_factory=OnLocal,
         description="A target node for this job to use for execution.",
         alias="runs-on",
     )
@@ -319,8 +317,8 @@ class Job(BaseModel):
         default_factory=list,
         description="A list of Stage of this job.",
     )
-    trigger_rule: TriggerRules = Field(
-        default=TriggerRules.all_success,
+    trigger_rule: Rule = Field(
+        default=Rule.ALL_SUCCESS,
         description=(
             "A trigger rule of tracking needed jobs if feature will use when "
             "the `raise_error` did not set from job and stage executions."
@@ -421,27 +419,27 @@ class Job(BaseModel):
             return WAIT
         elif all("skipped" in need_exist[job] for job in need_exist):
             return SKIP
-        elif self.trigger_rule == TriggerRules.all_done:
+        elif self.trigger_rule == Rule.ALL_DONE:
             return SUCCESS
-        elif self.trigger_rule == TriggerRules.all_success:
+        elif self.trigger_rule == Rule.ALL_SUCCESS:
             rs = all(
                 k not in need_exist[job]
                 for k in ("errors", "skipped")
                 for job in need_exist
             )
-        elif self.trigger_rule == TriggerRules.all_failed:
+        elif self.trigger_rule == Rule.ALL_FAILED:
             rs = all("errors" in need_exist[job] for job in need_exist)
-        elif self.trigger_rule == TriggerRules.one_success:
+        elif self.trigger_rule == Rule.ONE_SUCCESS:
             rs = sum(
                 k not in need_exist[job]
                 for k in ("errors", "skipped")
                 for job in need_exist
             ) + 1 == len(self.needs)
-        elif self.trigger_rule == TriggerRules.one_failed:
+        elif self.trigger_rule == Rule.ONE_FAILED:
             rs = sum("errors" in need_exist[job] for job in need_exist) == 1
-        elif self.trigger_rule == TriggerRules.none_skipped:
+        elif self.trigger_rule == Rule.NONE_SKIPPED:
             rs = all("skipped" not in need_exist[job] for job in need_exist)
-        elif self.trigger_rule == TriggerRules.none_failed:
+        elif self.trigger_rule == Rule.NONE_FAILED:
             rs = all("errors" not in need_exist[job] for job in need_exist)
         else:  # pragma: no cov
             raise NotImplementedError(
@@ -480,10 +478,16 @@ class Job(BaseModel):
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
             return not rs
-        except Exception as err:
-            raise JobException(f"{err.__class__.__name__}: {err}") from err
+        except Exception as e:
+            raise JobException(f"{e.__class__.__name__}: {e}") from e
 
-    def set_outputs(self, output: DictData, to: DictData) -> DictData:
+    def set_outputs(
+        self,
+        output: DictData,
+        to: DictData,
+        *,
+        job_id: Optional[None] = None,
+    ) -> DictData:
         """Set an outputs from execution process to the received context. The
         result from execution will pass to value of `strategies` key.
 
@@ -511,22 +515,21 @@ class Job(BaseModel):
 
         :param output: An output context.
         :param to: A context data that want to add output result.
+        :param job_id: A job ID if the id field does not set.
 
         :rtype: DictData
         """
         if "jobs" not in to:
             to["jobs"] = {}
 
-        if self.id is None and not dynamic(
-            "job_default_id", extras=self.extras
-        ):
+        if self.id is None and job_id is None:
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
 
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
-        _id: str = self.id or str(len(to["jobs"]) + 1)
+        _id: str = self.id or job_id
 
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}
@@ -550,6 +553,7 @@ class Job(BaseModel):
         parent_run_id: str | None = None,
         result: Result | None = None,
         event: Event | None = None,
+        raise_error: bool = True,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
@@ -562,6 +566,8 @@ class Job(BaseModel):
             data.
         :param event: (Event) An event manager that pass to the
             PoolThreadExecutor.
+        :param raise_error: (bool) A flag that all this method raise error to the
+            strategy execution. Default is `True`.
 
         :rtype: Result
         """
@@ -573,16 +579,18 @@ class Job(BaseModel):
             extras=self.extras,
         )
 
-        if self.runs_on.type == RunsOnType.LOCAL:
+        result.trace.info(f"[JOB]: Start execute job: {self.id!r}")
+        if self.runs_on.type == RunsOn.LOCAL:
             return local_execute(
                 job=self,
                 params=params,
                 result=result,
                 event=event,
+                raise_error=raise_error,
             )
-        elif self.runs_on.type == RunsOnType.SELF_HOSTED:  # pragma: no cov
+        elif self.runs_on.type == RunsOn.SELF_HOSTED:  # pragma: no cov
             pass
-        elif self.runs_on.type == RunsOnType.K8S:  # pragma: no cov
+        elif self.runs_on.type == RunsOn.K8S:  # pragma: no cov
             pass
 
         # pragma: no cov
@@ -603,7 +611,7 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
+    raise_error: bool = True,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -660,7 +668,7 @@ def local_execute_strategy(
                 context={
                     strategy_id: {
                         "matrix": strategy,
-                        "stages": context.pop("stages", {}),
+                        "stages": filter_func(context.pop("stages", {})),
                         "errors": JobException(error_msg).to_dict(),
                     },
                 },
@@ -684,21 +692,17 @@ def local_execute_strategy(
                     context={
                         strategy_id: {
                             "matrix": strategy,
-                            "stages": context.pop("stages", {}),
+                            "stages": filter_func(context.pop("stages", {})),
                             "errors": JobException(error_msg).to_dict(),
                         },
                     },
                 )
 
-        except (StageException, UtilException) as err:
-            result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            do_raise: bool = dynamic(
-                "job_raise_error", f=raise_error, extras=job.extras
-            )
-            if do_raise:
+        except (StageException, UtilException) as e:
+            result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
+            if raise_error:
                 raise JobException(
-                    f"Stage execution error: {err.__class__.__name__}: "
-                    f"{err}"
+                    f"Stage execution error: {e.__class__.__name__}: {e}"
                 ) from None
 
             return result.catch(
@@ -706,8 +710,8 @@ def local_execute_strategy(
                 context={
                     strategy_id: {
                         "matrix": strategy,
-                        "stages": context.pop("stages", {}),
-                        "errors": err.to_dict(),
+                        "stages": filter_func(context.pop("stages", {})),
+                        "errors": e.to_dict(),
                     },
                 },
             )
@@ -731,7 +735,7 @@ def local_execute(
     parent_run_id: str | None = None,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
+    raise_error: bool = True,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution or itself execution. It will generate matrix values at the first
@@ -748,7 +752,7 @@ def local_execute(
         data.
     :param event: (Event) An event manager that pass to the PoolThreadExecutor.
     :param raise_error: (bool) A flag that all this method raise error to the
-        strategy execution.
+        strategy execution. Default is `True`.
 
     :rtype: Result
     """
@@ -779,9 +783,9 @@ def local_execute(
                 )
 
             local_execute_strategy(
-                job=job,
-                strategy=strategy,
-                params=params,
+                job,
+                strategy,
+                params,
                 result=result,
                 event=event,
                 raise_error=raise_error,
@@ -851,13 +855,12 @@ def local_execute(
         for future in done:
             try:
                 future.result()
-            except JobException as err:
+            except JobException as e:
                 status = FAILED
                 result.trace.error(
-                    f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
-                    f"\n\t{err}"
+                    f"[JOB]: {ls} Catch:\n\t{e.__class__.__name__}:\n\t{e}"
                 )
-                context.update({"errors": err.to_dict()})
+                context.update({"errors": e.to_dict()})
 
     return result.catch(status=status, context=context)
 
@@ -870,7 +873,7 @@ def self_hosted_execute(
     parent_run_id: str | None = None,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
+    raise_error: bool = True,
 ) -> Result:  # pragma: no cov
     """Self-Hosted job execution with passing dynamic parameters from the
     workflow execution or itself execution. It will make request to the
@@ -924,10 +927,7 @@ def self_hosted_execute(
         return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
     if resp.status_code != 200:
-        do_raise: bool = dynamic(
-            "job_raise_error", f=raise_error, extras=job.extras
-        )
-        if do_raise:
+        if raise_error:
             raise JobException(
                 f"Job execution error from request to self-hosted: "
                 f"{job.runs_on.args.host!r}"
@@ -935,3 +935,59 @@ def self_hosted_execute(
 
         return result.catch(status=FAILED)
     return result.catch(status=SUCCESS)
+
+
+def azure_batch_execute(
+    job: Job,
+    params: DictData,
+    *,
+    run_id: str | None = None,
+    parent_run_id: str | None = None,
+    result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool | None = None,
+) -> Result:  # pragma no cov
+    """Azure Batch job execution that will run all job's stages on the Azure
+    Batch Node and extract the result file to be returning context result.
+
+    Steps:
+        - Create a Batch account and a Batch pool.
+        - Create a Batch job and add tasks to the job. Each task represents a
+          command to run on a compute node.
+        - Specify the command to run the Python script in the task. You can use
+          the cmd /c command to run the script with the Python interpreter.
+        - Upload the Python script and any required input files to Azure Storage
+          Account.
+        - Configure the task to download the input files from Azure Storage to
+          the compute node before running the script.
+        - Monitor the job and retrieve the output files from Azure Storage.
+
+    :param job:
+    :param params:
+    :param run_id:
+    :param parent_run_id:
+    :param result:
+    :param event:
+    :param raise_error:
+    :return:
+    """
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+        extras=job.extras,
+    )
+    if event and event.is_set():
+        return result.catch(
+            status=FAILED,
+            context={
+                "errors": JobException(
+                    "Job azure-batch execution was canceled from event that "
+                    "had set before start execution."
+                ).to_dict()
+            },
+        )
+    print(params)
+    print(raise_error)
+    return result.catch(status=SUCCESS)