ddeutil-workflow 0.0.49__py3-none-any.whl → 0.0.50__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.49"
+__version__: str = "0.0.50"

ddeutil/workflow/__init__.py

@@ -49,6 +49,7 @@ from .params import (
     StrParam,
 )
 from .result import (
+    CANCEL,
     FAILED,
     SKIP,
     SUCCESS,
@@ -101,6 +102,9 @@ from .utils import (
     get_diff_sec,
     get_dt_now,
     make_exec,
+    reach_next_minute,
+    replace_sec,
+    wait_to_next_minute,
 )
 from .workflow import (
     Release,

ddeutil/workflow/conf.py

@@ -53,7 +53,6 @@ class Config:  # pragma: no cov
         The config value can change when you call that config property again.
     """
 
-    # NOTE: Core
     @property
     def conf_path(self) -> Path:
         """Config path that keep all workflow template YAML files.
@@ -73,9 +72,13 @@ class Config:  # pragma: no cov
 
     @property
     def generate_id_simple_mode(self) -> bool:
+        """Flag for generate running ID with simple mode. That does not use
+        `md5` function after generate simple mode.
+
+        :rtype: bool
+        """
         return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
-    # NOTE: Register
     @property
     def registry_caller(self) -> list[str]:
         """Register Caller that is a list of importable string for the call
@@ -98,13 +101,16 @@ class Config:  # pragma: no cov
         )
         return [r.strip() for r in regis_filter_str.split(",")]
 
-    # NOTE: Log
     @property
     def trace_path(self) -> Path:
         return Path(env("LOG_TRACE_PATH", "./logs"))
 
     @property
     def debug(self) -> bool:
+        """Debug flag for echo log that use DEBUG mode.
+
+        :rtype: bool
+        """
         return str2bool(env("LOG_DEBUG_MODE", "true"))
 
     @property
@@ -144,7 +150,6 @@ class Config:  # pragma: no cov
     def log_datetime_format(self) -> str:
         return env("LOG_DATETIME_FORMAT", "%Y-%m-%d %H:%M:%S")
 
-    # NOTE: Stage
     @property
     def stage_raise_error(self) -> bool:
         return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
@@ -153,11 +158,6 @@ class Config:  # pragma: no cov
     def stage_default_id(self) -> bool:
         return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    # NOTE: Job
-    @property
-    def job_raise_error(self) -> bool:
-        return str2bool(env("CORE_JOB_RAISE_ERROR", "true"))
-
     @property
     def max_cron_per_workflow(self) -> int:
         """The maximum on value that store in workflow model.

ddeutil/workflow/exceptions.py

@@ -39,6 +39,9 @@ class BaseWorkflowException(Exception):
 class UtilException(BaseWorkflowException): ...
 
 
+class ResultException(UtilException): ...
+
+
 class StageException(BaseWorkflowException): ...
 
 
@@ -48,9 +51,6 @@ class JobException(BaseWorkflowException): ...
 class WorkflowException(BaseWorkflowException): ...
 
 
-class WorkflowFailException(WorkflowException): ...
-
-
 class ParamValueException(WorkflowException): ...
 
 

ddeutil/workflow/job.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
 mean the job model able to define `runs-on` key that allow you to run this
@@ -33,7 +32,6 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import dynamic
 from .exceptions import (
     JobException,
     StageException,
@@ -51,11 +49,12 @@ MatrixFilter = list[dict[str, Union[str, int]]]
 __all__: TupleStr = (
     "Strategy",
     "Job",
-    "TriggerRules",
+    "Rule",
     "RunsOn",
-    "RunsOnLocal",
-    "RunsOnSelfHosted",
-    "RunsOnK8s",
+    "RunsOnModel",
+    "OnLocal",
+    "OnSelfHosted",
+    "OnK8s",
     "make",
     "local_execute_strategy",
     "local_execute",
@@ -194,24 +193,25 @@ class Strategy(BaseModel):
         return make(self.matrix, self.include, self.exclude)
 
 
-class TriggerRules(str, Enum):
+class Rule(str, Enum):
     """Trigger rules enum object."""
 
-    all_success: str = "all_success"
-    all_failed: str = "all_failed"
-    all_done: str = "all_done"
-    one_failed: str = "one_failed"
-    one_success: str = "one_success"
-    none_failed: str = "none_failed"
-    none_skipped: str = "none_skipped"
+    ALL_SUCCESS: str = "all_success"
+    ALL_FAILED: str = "all_failed"
+    ALL_DONE: str = "all_done"
+    ONE_FAILED: str = "one_failed"
+    ONE_SUCCESS: str = "one_success"
+    NONE_FAILED: str = "none_failed"
+    NONE_SKIPPED: str = "none_skipped"
 
 
-class RunsOnType(str, Enum):
+class RunsOn(str, Enum):
     """Runs-On enum object."""
 
     LOCAL: str = "local"
     SELF_HOSTED: str = "self_hosted"
     K8S: str = "k8s"
+    AZ_BATCH: str = "azure_batch"
 
 
 class BaseRunsOn(BaseModel):  # pragma: no cov
@@ -221,47 +221,45 @@ class BaseRunsOn(BaseModel):  # pragma: no cov
 
     model_config = ConfigDict(use_enum_values=True)
 
-    type: Literal[RunsOnType.LOCAL]
+    type: Literal[RunsOn.LOCAL]
     args: DictData = Field(
         default_factory=dict,
         alias="with",
     )
 
 
-class RunsOnLocal(BaseRunsOn):  # pragma: no cov
+class OnLocal(BaseRunsOn):  # pragma: no cov
     """Runs-on local."""
 
-    type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
+    type: Literal[RunsOn.LOCAL] = Field(default=RunsOn.LOCAL)
 
 
 class SelfHostedArgs(BaseModel):
     host: str
 
 
-class RunsOnSelfHosted(BaseRunsOn):  # pragma: no cov
+class OnSelfHosted(BaseRunsOn):  # pragma: no cov
     """Runs-on self-hosted."""
 
-    type: Literal[RunsOnType.SELF_HOSTED] = Field(
-        default=RunsOnType.SELF_HOSTED
-    )
+    type: Literal[RunsOn.SELF_HOSTED] = Field(default=RunsOn.SELF_HOSTED)
     args: SelfHostedArgs = Field(alias="with")
 
 
-class RunsOnK8s(BaseRunsOn):  # pragma: no cov
+class OnK8s(BaseRunsOn):  # pragma: no cov
     """Runs-on Kubernetes."""
 
-    type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
+    type: Literal[RunsOn.K8S] = Field(default=RunsOn.K8S)
 
 
 def get_discriminator_runs_on(model: dict[str, Any]) -> str:
     return model.get("type", "local")
 
 
-RunsOn = Annotated[
+RunsOnModel = Annotated[
     Union[
-        Annotated[RunsOnK8s, Tag(RunsOnType.K8S)],
-        Annotated[RunsOnSelfHosted, Tag(RunsOnType.SELF_HOSTED)],
-        Annotated[RunsOnLocal, Tag(RunsOnType.LOCAL)],
+        Annotated[OnK8s, Tag(RunsOn.K8S)],
+        Annotated[OnSelfHosted, Tag(RunsOn.SELF_HOSTED)],
+        Annotated[OnLocal, Tag(RunsOn.LOCAL)],
     ],
     Discriminator(get_discriminator_runs_on),
 ]
@@ -305,8 +303,8 @@ class Job(BaseModel):
         default=None,
         description="A job description that can be string of markdown content.",
     )
-    runs_on: RunsOn = Field(
-        default_factory=RunsOnLocal,
+    runs_on: RunsOnModel = Field(
+        default_factory=OnLocal,
         description="A target node for this job to use for execution.",
         alias="runs-on",
     )
@@ -319,8 +317,8 @@ class Job(BaseModel):
         default_factory=list,
         description="A list of Stage of this job.",
     )
-    trigger_rule: TriggerRules = Field(
-        default=TriggerRules.all_success,
+    trigger_rule: Rule = Field(
+        default=Rule.ALL_SUCCESS,
         description=(
             "A trigger rule of tracking needed jobs if feature will use when "
             "the `raise_error` did not set from job and stage executions."
@@ -421,27 +419,27 @@ class Job(BaseModel):
             return WAIT
         elif all("skipped" in need_exist[job] for job in need_exist):
             return SKIP
-        elif self.trigger_rule == TriggerRules.all_done:
+        elif self.trigger_rule == Rule.ALL_DONE:
             return SUCCESS
-        elif self.trigger_rule == TriggerRules.all_success:
+        elif self.trigger_rule == Rule.ALL_SUCCESS:
             rs = all(
                 k not in need_exist[job]
                 for k in ("errors", "skipped")
                 for job in need_exist
             )
-        elif self.trigger_rule == TriggerRules.all_failed:
+        elif self.trigger_rule == Rule.ALL_FAILED:
             rs = all("errors" in need_exist[job] for job in need_exist)
-        elif self.trigger_rule == TriggerRules.one_success:
+        elif self.trigger_rule == Rule.ONE_SUCCESS:
             rs = sum(
                 k not in need_exist[job]
                 for k in ("errors", "skipped")
                 for job in need_exist
             ) + 1 == len(self.needs)
-        elif self.trigger_rule == TriggerRules.one_failed:
+        elif self.trigger_rule == Rule.ONE_FAILED:
             rs = sum("errors" in need_exist[job] for job in need_exist) == 1
-        elif self.trigger_rule == TriggerRules.none_skipped:
+        elif self.trigger_rule == Rule.NONE_SKIPPED:
             rs = all("skipped" not in need_exist[job] for job in need_exist)
-        elif self.trigger_rule == TriggerRules.none_failed:
+        elif self.trigger_rule == Rule.NONE_FAILED:
             rs = all("errors" not in need_exist[job] for job in need_exist)
         else:  # pragma: no cov
             raise NotImplementedError(
@@ -480,8 +478,8 @@ class Job(BaseModel):
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
             return not rs
-        except Exception as err:
-            raise JobException(f"{err.__class__.__name__}: {err}") from err
+        except Exception as e:
+            raise JobException(f"{e.__class__.__name__}: {e}") from e
 
     def set_outputs(
         self,
@@ -555,6 +553,7 @@ class Job(BaseModel):
         parent_run_id: str | None = None,
         result: Result | None = None,
         event: Event | None = None,
+        raise_error: bool = True,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
@@ -567,6 +566,8 @@ class Job(BaseModel):
             data.
         :param event: (Event) An event manager that pass to the
             PoolThreadExecutor.
+        :param raise_error: (bool) A flag that all this method raise error to the
+            strategy execution. Default is `True`.
 
         :rtype: Result
         """
@@ -578,16 +579,18 @@ class Job(BaseModel):
             extras=self.extras,
         )
 
-        if self.runs_on.type == RunsOnType.LOCAL:
+        result.trace.info(f"[JOB]: Start execute job: {self.id!r}")
+        if self.runs_on.type == RunsOn.LOCAL:
             return local_execute(
                 job=self,
                 params=params,
                 result=result,
                 event=event,
+                raise_error=raise_error,
             )
-        elif self.runs_on.type == RunsOnType.SELF_HOSTED:  # pragma: no cov
+        elif self.runs_on.type == RunsOn.SELF_HOSTED:  # pragma: no cov
             pass
-        elif self.runs_on.type == RunsOnType.K8S:  # pragma: no cov
+        elif self.runs_on.type == RunsOn.K8S:  # pragma: no cov
             pass
 
         # pragma: no cov
@@ -608,7 +611,7 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
+    raise_error: bool = True,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -665,7 +668,7 @@ def local_execute_strategy(
                 context={
                     strategy_id: {
                         "matrix": strategy,
-                        "stages": context.pop("stages", {}),
+                        "stages": filter_func(context.pop("stages", {})),
                         "errors": JobException(error_msg).to_dict(),
                     },
                 },
@@ -689,21 +692,17 @@ def local_execute_strategy(
                     context={
                         strategy_id: {
                             "matrix": strategy,
-                            "stages": context.pop("stages", {}),
+                            "stages": filter_func(context.pop("stages", {})),
                             "errors": JobException(error_msg).to_dict(),
                         },
                     },
                 )
 
-        except (StageException, UtilException) as err:
-            result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            do_raise: bool = dynamic(
-                "job_raise_error", f=raise_error, extras=job.extras
-            )
-            if do_raise:
+        except (StageException, UtilException) as e:
+            result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
+            if raise_error:
                 raise JobException(
-                    f"Stage execution error: {err.__class__.__name__}: "
-                    f"{err}"
+                    f"Stage execution error: {e.__class__.__name__}: {e}"
                 ) from None
 
             return result.catch(
@@ -711,8 +710,8 @@ def local_execute_strategy(
                 context={
                     strategy_id: {
                         "matrix": strategy,
-                        "stages": context.pop("stages", {}),
-                        "errors": err.to_dict(),
+                        "stages": filter_func(context.pop("stages", {})),
+                        "errors": e.to_dict(),
                     },
                 },
             )
@@ -736,7 +735,7 @@ def local_execute(
     parent_run_id: str | None = None,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
+    raise_error: bool = True,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution or itself execution. It will generate matrix values at the first
@@ -753,7 +752,7 @@ def local_execute(
         data.
     :param event: (Event) An event manager that pass to the PoolThreadExecutor.
     :param raise_error: (bool) A flag that all this method raise error to the
-        strategy execution.
+        strategy execution. Default is `True`.
 
     :rtype: Result
     """
@@ -784,9 +783,9 @@ def local_execute(
                 )
 
             local_execute_strategy(
-                job=job,
-                strategy=strategy,
-                params=params,
+                job,
+                strategy,
+                params,
                 result=result,
                 event=event,
                 raise_error=raise_error,
@@ -856,13 +855,12 @@ def local_execute(
         for future in done:
             try:
                 future.result()
-            except JobException as err:
+            except JobException as e:
                 status = FAILED
                 result.trace.error(
-                    f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
-                    f"\n\t{err}"
+                    f"[JOB]: {ls} Catch:\n\t{e.__class__.__name__}:\n\t{e}"
                 )
-                context.update({"errors": err.to_dict()})
+                context.update({"errors": e.to_dict()})
 
     return result.catch(status=status, context=context)
 
@@ -875,7 +873,7 @@ def self_hosted_execute(
     parent_run_id: str | None = None,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
+    raise_error: bool = True,
 ) -> Result:  # pragma: no cov
     """Self-Hosted job execution with passing dynamic parameters from the
     workflow execution or itself execution. It will make request to the
@@ -929,10 +927,7 @@ def self_hosted_execute(
         return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
     if resp.status_code != 200:
-        do_raise: bool = dynamic(
-            "job_raise_error", f=raise_error, extras=job.extras
-        )
-        if do_raise:
+        if raise_error:
             raise JobException(
                 f"Job execution error from request to self-hosted: "
                 f"{job.runs_on.args.host!r}"
@@ -940,3 +935,59 @@ def self_hosted_execute(
 
         return result.catch(status=FAILED)
     return result.catch(status=SUCCESS)
+
+
+def azure_batch_execute(
+    job: Job,
+    params: DictData,
+    *,
+    run_id: str | None = None,
+    parent_run_id: str | None = None,
+    result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool | None = None,
+) -> Result:  # pragma no cov
+    """Azure Batch job execution that will run all job's stages on the Azure
+    Batch Node and extract the result file to be returning context result.
+
+    Steps:
+        - Create a Batch account and a Batch pool.
+        - Create a Batch job and add tasks to the job. Each task represents a
+          command to run on a compute node.
+        - Specify the command to run the Python script in the task. You can use
+          the cmd /c command to run the script with the Python interpreter.
+        - Upload the Python script and any required input files to Azure Storage
+          Account.
+        - Configure the task to download the input files from Azure Storage to
+          the compute node before running the script.
+        - Monitor the job and retrieve the output files from Azure Storage.
+
+    :param job:
+    :param params:
+    :param run_id:
+    :param parent_run_id:
+    :param result:
+    :param event:
+    :param raise_error:
+    :return:
+    """
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+        extras=job.extras,
+    )
+    if event and event.is_set():
+        return result.catch(
+            status=FAILED,
+            context={
+                "errors": JobException(
+                    "Job azure-batch execution was canceled from event that "
+                    "had set before start execution."
+                ).to_dict()
+            },
+        )
+    print(params)
+    print(raise_error)
+    return result.catch(status=SUCCESS)

ddeutil/workflow/logs.py

@@ -394,7 +394,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
         """
         cut_run_id: str = cut_id(self.run_id)
         if not self.parent_run_id:
-            return f"{cut_run_id} -> {' ' * 6}"
+            return f"{cut_run_id}"
 
         cut_parent_run_id: str = cut_id(self.parent_run_id)
         return f"{cut_parent_run_id} -> {cut_run_id}"

ddeutil/workflow/result.py

@@ -21,6 +21,7 @@ from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
+from .exceptions import ResultException
 from .logs import Trace, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
@@ -34,12 +35,14 @@ class Status(IntEnum):
     FAILED: int = 1
     WAIT: int = 2
     SKIP: int = 3
+    CANCEL: int = 4
 
 
 SUCCESS = Status.SUCCESS
 FAILED = Status.FAILED
 WAIT = Status.WAIT
 SKIP = Status.SKIP
+CANCEL = Status.CANCEL
 
 
 @dataclass(
@@ -63,7 +66,6 @@ class Result:
 
     status: Status = field(default=WAIT)
     context: DictData = field(default_factory=dict)
-    errors: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
@@ -137,7 +139,7 @@ class Result:
         self,
         status: int | Status,
         context: DictData | None = None,
-        error: DictData | None = None,
+        **kwargs,
     ) -> Self:
         """Catch the status and context to this Result object. This method will
         use between a child execution return a result, and it wants to pass
@@ -145,7 +147,6 @@ class Result:
 
         :param status: A status enum object.
         :param context: A context data that will update to the current context.
-        :param error: An error data that will update to the current errors.
 
         :rtype: Self
         """
@@ -153,7 +154,14 @@ class Result:
             Status(status) if isinstance(status, int) else status
         )
         self.__dict__["context"].update(context or {})
-        self.__dict__["errors"].update(error or {})
+        if kwargs:
+            for k in kwargs:
+                if k in self.__dict__["context"]:
+                    self.__dict__["context"][k].update(kwargs[k])
+                else:
+                    raise ResultException(
+                        f"The key {k!r} does not exists on context data."
+                    )
         return self
 
     def alive_time(self) -> float:  # pragma: no cov