ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.51__py3-none-any.whl

ddeutil/workflow/job.py

@@ -27,18 +27,18 @@ from threading import Event
 from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, SecretStr, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, Matrix, TupleStr
+from .__types import DictData, DictStr, Matrix
 from .exceptions import (
     JobException,
     StageException,
     UtilException,
     to_dict,
 )
-from .result import FAILED, SKIP, SUCCESS, WAIT, Result, Status
+from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
 from .stages import Stage
 from .utils import cross_product, filter_func, gen_id
@@ -46,21 +46,6 @@ from .utils import cross_product, filter_func, gen_id
 MatrixFilter = list[dict[str, Union[str, int]]]
 
 
-__all__: TupleStr = (
-    "Strategy",
-    "Job",
-    "Rule",
-    "RunsOn",
-    "RunsOnModel",
-    "OnLocal",
-    "OnSelfHosted",
-    "OnK8s",
-    "make",
-    "local_execute_strategy",
-    "local_execute",
-)
-
-
 @freeze_args
 @lru_cache
 def make(
@@ -120,7 +105,6 @@ def make(
 
         add.append(inc)
 
-    # NOTE: Merge all matrix together.
     final.extend(add)
     return final
 
@@ -210,8 +194,8 @@ class RunsOn(str, Enum):
 
     LOCAL: str = "local"
     SELF_HOSTED: str = "self_hosted"
-    K8S: str = "k8s"
     AZ_BATCH: str = "azure_batch"
+    DOCKER: str = "docker"
 
 
 class BaseRunsOn(BaseModel):  # pragma: no cov
@@ -221,10 +205,14 @@ class BaseRunsOn(BaseModel):  # pragma: no cov
 
     model_config = ConfigDict(use_enum_values=True)
 
-    type: Literal[RunsOn.LOCAL]
+    type: RunsOn = Field(description="A runs-on type.")
     args: DictData = Field(
         default_factory=dict,
         alias="with",
+        description=(
+            "An argument that pass to the runs-on execution function. This "
+            "args will override by this child-model with specific args model."
+        ),
     )
 
 
@@ -235,7 +223,9 @@ class OnLocal(BaseRunsOn):  # pragma: no cov
 
 
 class SelfHostedArgs(BaseModel):
-    host: str
+    """Self-Hosted arguments."""
+
+    host: str = Field(description="A host URL of the target self-hosted.")
 
 
 class OnSelfHosted(BaseRunsOn):  # pragma: no cov
@@ -245,20 +235,48 @@ class OnSelfHosted(BaseRunsOn):  # pragma: no cov
     args: SelfHostedArgs = Field(alias="with")
 
 
-class OnK8s(BaseRunsOn):  # pragma: no cov
-    """Runs-on Kubernetes."""
+class AzBatchArgs(BaseModel):
+    batch_account_name: str
+    batch_account_key: SecretStr
+    batch_account_url: str
+    storage_account_name: str
+    storage_account_key: SecretStr
+
+
+class OnAzBatch(BaseRunsOn):  # pragma: no cov
+
+    type: Literal[RunsOn.AZ_BATCH] = Field(default=RunsOn.AZ_BATCH)
+    args: AzBatchArgs = Field(alias="with")
+
+
+class DockerArgs(BaseModel):
+    image: str = Field(
+        default="ubuntu-latest",
+        description=(
+            "An image that want to run like `ubuntu-22.04`, `windows-latest`, "
+            ", `ubuntu-24.04-arm`, or `macos-14`"
+        ),
+    )
+    env: DictData = Field(default_factory=dict)
+    volume: DictData = Field(default_factory=dict)
+
 
-    type: Literal[RunsOn.K8S] = Field(default=RunsOn.K8S)
+class OnDocker(BaseRunsOn):  # pragma: no cov
+    """Runs-on Docker container."""
+
+    type: Literal[RunsOn.DOCKER] = Field(default=RunsOn.DOCKER)
+    args: DockerArgs = Field(alias="with", default_factory=DockerArgs)
 
 
 def get_discriminator_runs_on(model: dict[str, Any]) -> str:
+    """Get discriminator of the RunsOn models."""
     return model.get("type", "local")
 
 
 RunsOnModel = Annotated[
     Union[
-        Annotated[OnK8s, Tag(RunsOn.K8S)],
         Annotated[OnSelfHosted, Tag(RunsOn.SELF_HOSTED)],
+        Annotated[OnDocker, Tag(RunsOn.DOCKER)],
         Annotated[OnLocal, Tag(RunsOn.LOCAL)],
     ],
     Discriminator(get_discriminator_runs_on),
@@ -288,7 +306,6 @@ class Job(BaseModel):
         ...             "name": "Some stage",
         ...             "run": "print('Hello World')",
         ...         },
-        ...         ...
         ...     ],
         ... }
     """
@@ -327,7 +344,7 @@ class Job(BaseModel):
     )
     needs: list[str] = Field(
         default_factory=list,
-        description="A list of the job ID that want to run before this job.",
+        description="A list of the job that want to run before this job model.",
     )
     strategy: Strategy = Field(
         default_factory=Strategy,
@@ -359,7 +376,7 @@ class Job(BaseModel):
             name: str = stage.iden
             if name in rs:
                 raise ValueError(
-                    "Stage name in jobs object should not be duplicate."
+                    f"Stage name, {name!r}, should not be duplicate."
                 )
             rs.append(name)
         return value
@@ -372,7 +389,9 @@ class Job(BaseModel):
         """
         # VALIDATE: Validate job id should not dynamic with params template.
         if has_template(self.id):
-            raise ValueError("Job ID should not has any template.")
+            raise ValueError(
+                f"Job ID, {self.id!r}, should not has any template."
+            )
 
         return self
 
@@ -390,7 +409,7 @@ class Job(BaseModel):
                 if self.extras:
                     stage.extras = self.extras
                 return stage
-        raise ValueError(f"Stage ID {stage_id} does not exists")
+        raise ValueError(f"Stage {stage_id!r} does not exists in this job.")
 
     def check_needs(
         self,
@@ -497,14 +516,14 @@ class Job(BaseModel):
             ... (i)   output: {'strategy-01': bar, 'strategy-02': bar}
             ... (ii)  to: {'jobs': {}}
 
-        The result of the `to` variable will be;
+        The result of the `to` argument will be;
 
             ... (iii) to: {
                         'jobs': {
                             '<job-id>': {
                                 'strategies': {
                                     'strategy-01': bar,
-                                    'strategy-02': bar
+                                    'strategy-02': bar,
                                 }
                             }
                         }
@@ -527,22 +546,27 @@ class Job(BaseModel):
                 "This job do not set the ID before setting execution output."
             )
 
-        # NOTE: If the job ID did not set, it will use index of jobs key
-        #   instead.
         _id: str = self.id or job_id
-
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
+        skipping: dict[str, bool] = (
+            {"skipped": output.pop("skipped", False)}
+            if "skipped" in output
+            else {}
+        )
 
-        if "SKIP" in output:  # pragma: no cov
-            to["jobs"][_id] = output["SKIP"]
-        elif self.strategy.is_set():
-            to["jobs"][_id] = {"strategies": output, **errors}
+        if self.strategy.is_set():
+            to["jobs"][_id] = {"strategies": output, **skipping, **errors}
+        elif len(k := output.keys()) > 1:  # pragma: no cov
+            raise JobException(
+                "Strategy output from execution return more than one ID while "
+                "this job does not set strategy."
+            )
         else:
-            _output = output.get(next(iter(output), "FIRST"), {})
+            _output: DictData = {} if len(k) == 0 else output[list(k)[0]]
             _output.pop("matrix", {})
-            to["jobs"][_id] = {**_output, **errors}
+            to["jobs"][_id] = {**_output, **skipping, **errors}
         return to
 
     def execute(
@@ -559,6 +583,9 @@ class Job(BaseModel):
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the `stages` field of this job.
 
+            This method be execution routing for call dynamic execution function
+        with specific target `runs-on` value.
+
         :param params: An input parameters that use on job execution.
         :param run_id: (str) A job running ID.
         :param parent_run_id: (str) A parent workflow running ID.
@@ -566,8 +593,10 @@ class Job(BaseModel):
             data.
         :param event: (Event) An event manager that pass to the
             PoolThreadExecutor.
-        :param raise_error: (bool) A flag that all this method raise error to the
-            strategy execution. Default is `True`.
+        :param raise_error: (bool) A flag that all this method raise error to
+            the strategy execution. Default is `True`.
+
+        :raise NotImplementedError: If the `runs-on` value does not implement.
 
         :rtype: Result
         """
@@ -579,28 +608,36 @@ class Job(BaseModel):
             extras=self.extras,
         )
 
-        result.trace.info(f"[JOB]: Start execute job: {self.id!r}")
+        result.trace.info(
+            f"[JOB]: Execute: {self.id!r} on {self.runs_on.type!r}"
+        )
         if self.runs_on.type == RunsOn.LOCAL:
             return local_execute(
-                job=self,
-                params=params,
-                result=result,
+                self,
+                params,
+                run_id=run_id,
+                parent_run_id=parent_run_id,
                 event=event,
                 raise_error=raise_error,
             )
         elif self.runs_on.type == RunsOn.SELF_HOSTED:  # pragma: no cov
             pass
-        elif self.runs_on.type == RunsOn.K8S:  # pragma: no cov
-            pass
+        elif self.runs_on.type == RunsOn.DOCKER:  # pragma: no cov
+            docker_execution(
+                self,
+                params,
+                run_id=run_id,
+                parent_run_id=parent_run_id,
+                event=event,
+                raise_error=raise_error,
+            )
 
         # pragma: no cov
         result.trace.error(
-            f"[JOB]: Job executor does not support for runs-on type: "
-            f"{self.runs_on.type} yet"
+            f"[JOB]: Execution not support runs-on: {self.runs_on.type!r} yet."
         )
         raise NotImplementedError(
-            f"The job runs-on other type: {self.runs_on.type} does not "
-            f"support yet."
+            f"Execution runs-on type: {self.runs_on.type} does not support yet."
         )
 
 
@@ -622,6 +659,8 @@ def local_execute_strategy(
 
         The result of this execution will return result with strategy ID
     that generated from the `gen_id` function with an input strategy value.
+    For each stage that execution with this strategy metrix, it will use the
+    `set_outputs` method for reconstruct result context data.
 
     :raise JobException: If it has any error from `StageException` or
         `UtilException`.
@@ -645,8 +684,10 @@ def local_execute_strategy(
     context.update({"matrix": strategy, "stages": {}})
 
     if strategy:
-        result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
-        result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
+        result.trace.info(f"[JOB]: Execute Strategy: {strategy_id!r}")
+        result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
+    else:
+        result.trace.info("[JOB]: Execute Empty-Strategy")
 
     for stage in job.stages:
 
@@ -654,7 +695,7 @@ def local_execute_strategy(
             stage.extras = job.extras
 
         if stage.is_skipped(params=context):
-            result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
+            result.trace.info(f"[JOB]: Skip Stage: {stage.iden!r}")
             stage.set_outputs(output={"skipped": True}, to=context)
             continue
 
@@ -664,7 +705,7 @@ def local_execute_strategy(
                 "strategy execution."
             )
             return result.catch(
-                status=FAILED,
+                status=CANCEL,
                 context={
                     strategy_id: {
                         "matrix": strategy,
@@ -675,6 +716,7 @@ def local_execute_strategy(
             )
 
         try:
+            result.trace.info(f"[JOB]: Execute Stage: {stage.iden!r}")
             rs: Result = stage.handler_execute(
                 params=context,
                 run_id=result.run_id,
@@ -682,22 +724,6 @@ def local_execute_strategy(
                 event=event,
             )
             stage.set_outputs(rs.context, to=context)
-            if rs.status == FAILED:
-                error_msg: str = (
-                    f"Job strategy was break because it has a stage, "
-                    f"{stage.iden}, failed without raise error."
-                )
-                return result.catch(
-                    status=FAILED,
-                    context={
-                        strategy_id: {
-                            "matrix": strategy,
-                            "stages": filter_func(context.pop("stages", {})),
-                            "errors": JobException(error_msg).to_dict(),
-                        },
-                    },
-                )
-
         except (StageException, UtilException) as e:
             result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
             if raise_error:
@@ -716,6 +742,22 @@ def local_execute_strategy(
                 },
             )
 
+        if rs.status == FAILED:
+            error_msg: str = (
+                f"Job strategy was break because stage, {stage.iden}, "
+                f"failed without raise error."
+            )
+            return result.catch(
+                status=FAILED,
+                context={
+                    strategy_id: {
+                        "matrix": strategy,
+                        "stages": filter_func(context.pop("stages", {})),
+                        "errors": JobException(error_msg).to_dict(),
+                    },
+                },
+            )
+
     return result.catch(
         status=SUCCESS,
         context={
@@ -733,7 +775,6 @@ def local_execute(
     *,
     run_id: str | None = None,
     parent_run_id: str | None = None,
-    result: Result | None = None,
     event: Event | None = None,
     raise_error: bool = True,
 ) -> Result:
@@ -748,8 +789,6 @@ def local_execute(
     :param params: (DictData) An input parameters that use on job execution.
     :param run_id: (str) A job running ID for this execution.
     :param parent_run_id: (str) A parent workflow running ID for this release.
-    :param result: (Result) A result object for keeping context and status
-        data.
     :param event: (Event) An event manager that pass to the PoolThreadExecutor.
     :param raise_error: (bool) A flag that all this method raise error to the
         strategy execution. Default is `True`.
@@ -757,12 +796,12 @@ def local_execute(
     :rtype: Result
     """
     result: Result = Result.construct_with_rs_or_id(
-        result,
         run_id=run_id,
         parent_run_id=parent_run_id,
         id_logic=(job.id or "not-set"),
         extras=job.extras,
     )
+
     event: Event = Event() if event is None else event
 
     # NOTE: Normal Job execution without parallel strategy matrix. It uses
@@ -773,7 +812,7 @@ def local_execute(
 
             if event and event.is_set():  # pragma: no cov
                 return result.catch(
-                    status=FAILED,
+                    status=CANCEL,
                     context={
                         "errors": JobException(
                             "Job strategy was canceled from event that had set "
@@ -791,7 +830,7 @@ def local_execute(
                 raise_error=raise_error,
             )
 
-        return result.catch(status=result.status)
+        return result
 
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
@@ -802,7 +841,7 @@ def local_execute(
 
     if event and event.is_set():  # pragma: no cov
         return result.catch(
-            status=FAILED,
+            status=CANCEL,
             context={
                 "errors": JobException(
                     "Job strategy was canceled from event that had set "
@@ -841,7 +880,7 @@ def local_execute(
 
             if len(done) != len(futures):
                 result.trace.warning(
-                    "[JOB]: Set the event for stop running stage."
+                    "[JOB]: Set event for stop pending stage future."
                 )
                 event.set()
                 for future in not_done:
@@ -850,7 +889,7 @@ def local_execute(
             nd: str = (
                 f", the strategies do not run is {not_done}" if not_done else ""
             )
-            result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+            result.trace.debug(f"[JOB]: Strategy set Fail-Fast{nd}")
 
         for future in done:
             try:
@@ -871,7 +910,6 @@ def self_hosted_execute(
     *,
     run_id: str | None = None,
     parent_run_id: str | None = None,
-    result: Result | None = None,
     event: Event | None = None,
     raise_error: bool = True,
 ) -> Result:  # pragma: no cov
@@ -883,8 +921,6 @@ def self_hosted_execute(
     :param params: (DictData) An input parameters that use on job execution.
     :param run_id: (str) A job running ID for this execution.
     :param parent_run_id: (str) A parent workflow running ID for this release.
-    :param result: (Result) A result object for keeping context and status
-        data.
     :param event: (Event) An event manager that pass to the PoolThreadExecutor.
     :param raise_error: (bool) A flag that all this method raise error to the
         strategy execution.
@@ -892,7 +928,6 @@ def self_hosted_execute(
     :rtype: Result
     """
     result: Result = Result.construct_with_rs_or_id(
-        result,
         run_id=run_id,
         parent_run_id=parent_run_id,
         id_logic=(job.id or "not-set"),
@@ -901,7 +936,7 @@ def self_hosted_execute(
 
     if event and event.is_set():
         return result.catch(
-            status=FAILED,
+            status=CANCEL,
             context={
                 "errors": JobException(
                     "Job self-hosted execution was canceled from event that "
@@ -943,7 +978,6 @@ def azure_batch_execute(
     *,
     run_id: str | None = None,
     parent_run_id: str | None = None,
-    result: Result | None = None,
     event: Event | None = None,
     raise_error: bool | None = None,
 ) -> Result:  # pragma no cov
@@ -962,17 +996,19 @@ def azure_batch_execute(
           the compute node before running the script.
         - Monitor the job and retrieve the output files from Azure Storage.
 
+    References:
+        - https://docs.azure.cn/en-us/batch/tutorial-parallel-python
+
     :param job:
     :param params:
     :param run_id:
     :param parent_run_id:
-    :param result:
     :param event:
     :param raise_error:
-    :return:
+
+    :rtype: Result
     """
     result: Result = Result.construct_with_rs_or_id(
-        result,
         run_id=run_id,
         parent_run_id=parent_run_id,
         id_logic=(job.id or "not-set"),
@@ -980,7 +1016,7 @@ def azure_batch_execute(
     )
     if event and event.is_set():
         return result.catch(
-            status=FAILED,
+            status=CANCEL,
             context={
                 "errors": JobException(
                     "Job azure-batch execution was canceled from event that "
@@ -991,3 +1027,40 @@ def azure_batch_execute(
     print(params)
     print(raise_error)
     return result.catch(status=SUCCESS)
+
+
+def docker_execution(
+    job: Job,
+    params: DictData,
+    *,
+    run_id: str | None = None,
+    parent_run_id: str | None = None,
+    event: Event | None = None,
+    raise_error: bool | None = None,
+):
+    """Docker job execution.
+
+    Steps:
+        - Pull the image
+        - Install this workflow package
+        - Start push job to run to target Docker container.
+    """
+    result: Result = Result.construct_with_rs_or_id(
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+        extras=job.extras,
+    )
+    if event and event.is_set():
+        return result.catch(
+            status=CANCEL,
+            context={
+                "errors": JobException(
+                    "Job Docker execution was canceled from event that "
+                    "had set before start execution."
+                ).to_dict()
+            },
+        )
+    print(params)
+    print(raise_error)
+    return result.catch(status=SUCCESS)

ddeutil/workflow/logs.py

@@ -224,18 +224,30 @@ class BaseTrace(ABC):  # pragma: no cov
             "Adjust make message method for this trace object before using."
         )
 
-    def debug(self, message: str):
-        """Write trace log with append mode and logging this message with the
-        DEBUG level.
+    def __logging(
+        self, message: str, mode: str, *, is_err: bool = False
+    ) -> None:
+        """Write trace log with append mode and logging this message with any
+        logging level.
 
         :param message: (str) A message that want to log.
         """
         msg: str = self.make_message(message)
 
-        if dynamic("debug", extras=self.extras):
-            self.writer(msg)
+        if mode != "debug" or (
+            mode == "debug" and dynamic("debug", extras=self.extras)
+        ):
+            self.writer(msg, is_err=is_err)
+
+        getattr(logger, mode)(msg, stacklevel=3)
+
+    def debug(self, message: str):
+        """Write trace log with append mode and logging this message with the
+        DEBUG level.
 
-        logger.debug(msg, stacklevel=2)
+        :param message: (str) A message that want to log.
+        """
+        self.__logging(message, mode="debug")
 
     def info(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -243,9 +255,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        self.writer(msg)
-        logger.info(msg, stacklevel=2)
+        self.__logging(message, mode="info")
 
     def warning(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -253,9 +263,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        self.writer(msg)
-        logger.warning(msg, stacklevel=2)
+        self.__logging(message, mode="warning")
 
     def error(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -263,9 +271,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        self.writer(msg, is_err=True)
-        logger.error(msg, stacklevel=2)
+        self.__logging(message, mode="error", is_err=True)
 
     def exception(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -273,9 +279,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        self.writer(msg, is_err=True)
-        logger.exception(msg, stacklevel=2)
+        self.__logging(message, mode="exception", is_err=True)
 
     async def adebug(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with