ddeutil-workflow 0.0.84__py3-none-any.whl → 0.0.86__py3-none-any.whl

ddeutil/workflow/job.py

@@ -56,8 +56,10 @@ from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
+from . import JobSkipError
 from .__types import DictData, DictStr, Matrix, StrOrNone
-from .errors import JobCancelError, JobError, to_dict
+from .conf import pass_env
+from .errors import JobCancelError, JobError, mark_errors, to_dict
 from .result import (
     CANCEL,
     FAILED,
@@ -73,7 +75,7 @@ from .result import (
 from .reusables import has_template, param2template
 from .stages import Stage
 from .traces import Trace, get_trace
-from .utils import cross_product, extract_id, filter_func, gen_id
+from .utils import cross_product, extract_id, filter_func, gen_id, get_dt_now
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -452,41 +454,22 @@ class Job(BaseModel):
     execution, dependency management, conditional execution, and multienvironment
     deployment.
 
-    Jobs are the primary execution units within workflows, providing:
-    - Stage lifecycle management
-    - Execution environment abstraction
-    - Matrix strategy support for parallel execution
-    - Dependency resolution via job needs
-    - Output coordination between stages
-
-    Attributes:
-        id (str, optional): Unique job identifier within workflow
-        desc (str, optional): Job description in Markdown format
-        runs_on (RunsOnModel): Execution environment configuration
-        condition (str, optional): Conditional execution expression
-        stages (list[Stage]): Ordered list of stages to execute
-        trigger_rule (Rule): Rule for handling job dependencies
-        needs (list[str]): List of prerequisite job IDs
-        strategy (Strategy): Matrix strategy for parameterized execution
-        extras (dict): Additional configuration parameters
-
     Example:
-        ```python
-        job = Job(
-            id="data-processing",
-            desc="Process daily data files",
-            runs_on=OnLocal(),
-            stages=[
-                EmptyStage(name="Start", echo="Processing started"),
-                PyStage(name="Process", run="process_data()"),
-                EmptyStage(name="Complete", echo="Processing finished")
-            ],
-            strategy=Strategy(
-                matrix={'env': ['dev', 'prod']},
-                max_parallel=2
-            )
-        )
-        ```
+        >>> from ddeutil.workflow.stages import EmptyStage, PyStage
+        >>> job = Job(
+        ...     id="data-processing",
+        ...     desc="Process daily data files",
+        ...     runs_on=OnLocal(),
+        ...     stages=[
+        ...         EmptyStage(name="Start", echo="Processing started"),
+        ...         PyStage(name="Process", run="process_data()"),
+        ...         EmptyStage(name="Complete", echo="Processing finished")
+        ...     ],
+        ...     strategy=Strategy(
+        ...         matrix={'env': ['dev', 'prod']},
+        ...         max_parallel=2
+        ...     )
+        ... )
     """
 
     id: StrOrNone = Field(
@@ -514,6 +497,15 @@ class Job(BaseModel):
         default_factory=list,
         description="A list of Stage model of this job.",
     )
+    retry: int = Field(
+        default=0,
+        ge=0,
+        lt=20,
+        description=(
+            "A retry number if job route execution got the error exclude skip "
+            "and cancel exception class."
+        ),
+    )
     trigger_rule: Rule = Field(
         default=Rule.ALL_SUCCESS,
         validate_default=True,
@@ -751,7 +743,7 @@ class Job(BaseModel):
             #   should use the `re` module to validate eval-string before
             #   running.
             rs: bool = eval(
-                param2template(self.condition, params, extras=self.extras),
+                self.pass_template(self.condition, params),
                 globals() | params,
                 {},
             )
@@ -802,8 +794,9 @@ class Job(BaseModel):
         extract from the result context if it exists. If it does not found, it
         will not set on the received context.
 
-        :raise JobError: If the job's ID does not set and the setting
-            default job ID flag does not set.
+        Raises:
+            JobError: If the job's ID does not set and the setting default job
+                ID flag does not set.
 
         Args:
             output: (DictData) A result data context that want to extract
@@ -831,9 +824,14 @@ class Job(BaseModel):
         status: dict[str, Status] = (
             {"status": output.pop("status")} if "status" in output else {}
         )
+        info: DictData = (
+            {"info": output.pop("info")} if "info" in output else {}
+        )
         kwargs: DictData = kwargs or {}
         if self.strategy.is_set():
-            to["jobs"][_id] = {"strategies": output} | errors | status | kwargs
+            to["jobs"][_id] = (
+                {"strategies": output} | errors | status | info | kwargs
+            )
         elif len(k := output.keys()) > 1:  # pragma: no cov
             raise JobError(
                 "Strategy output from execution return more than one ID while "
@@ -842,7 +840,7 @@ class Job(BaseModel):
         else:
             _output: DictData = {} if len(k) == 0 else output[list(k)[0]]
             _output.pop("matrix", {})
-            to["jobs"][_id] = _output | errors | status | kwargs
+            to["jobs"][_id] = _output | errors | status | info | kwargs
         return to
 
     def get_outputs(
@@ -854,10 +852,12 @@ class Job(BaseModel):
         """Get the outputs from jobs data. It will get this job ID or passing
         custom ID from the job outputs mapping.
 
-        :param output: (DictData) A job outputs data that want to extract
-        :param job_id: (StrOrNone) A job ID if the `id` field does not set.
+        Args:
+            output (DictData): A job outputs data that want to extract
+            job_id (StrOrNone): A job ID if the `id` field does not set.
 
-        :rtype: DictData
+        Returns:
+            DictData: An output data.
         """
         _id: str = self.id or job_id
         if self.strategy.is_set():
@@ -865,141 +865,291 @@ class Job(BaseModel):
         else:
             return output.get("jobs", {}).get(_id, {})
 
-    def execute(
+    def pass_template(self, value: Any, params: DictData) -> Any:
+        """Pass template and environment variable to any value that can
+        templating.
+
+        Args:
+            value (Any): An any value.
+            params (DictData): A parameter data that want to use in this
+                execution.
+
+        Returns:
+            Any: A templated value.
+        """
+        return pass_env(param2template(value, params, extras=self.extras))
+
+    def process(
         self,
         params: DictData,
+        run_id: str,
+        context: DictData,
         *,
-        run_id: StrOrNone = None,
+        parent_run_id: Optional[str] = None,
         event: Optional[Event] = None,
     ) -> Result:
-        """Job execution with passing dynamic parameters from the workflow
-        execution. It will generate matrix values at the first step and run
-        multithread on this metrics to the `stages` field of this job.
-
-            This method be execution routing for call dynamic execution function
-        with specific target `runs-on` value.
+        """Process routing method that will route the provider function depend
+        on runs-on value.
 
-        Args
-            params: (DictData) A parameter context that also pass from the
-                workflow execute method.
-            run_id: (str) An execution running ID.
-            event: (Event) An Event manager instance that use to cancel this
-                execution if it forces stopped by parent execution.
+        Args:
+            params (DictData): A parameter data that want to use in this
+                execution.
+            run_id (str): A running stage ID.
+            context (DictData): A context data that was passed from handler
+                method.
+            parent_run_id (str, default None): A parent running ID.
+            event (Event, default None): An event manager that use to track
+                parent process was not force stopped.
 
-        Returns
-            Result: Return Result object that create from execution context.
+        Returns:
+            Result: The execution result with status and context data.
         """
-        ts: float = time.monotonic()
-        parent_run_id, run_id = extract_id(
-            (self.id or "EMPTY"), run_id=run_id, extras=self.extras
-        )
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         trace.info(
-            f"[JOB]: Routing for "
+            f"[JOB]: Routing "
             f"{''.join(self.runs_on.type.value.split('_')).title()}: "
             f"{self.id!r}"
         )
-
+        rs: Optional[Result] = None
         if self.runs_on.type == LOCAL:
-            return local_execute(
+            rs = local_process(
                 self,
                 params,
+                context=context,
                 run_id=parent_run_id,
                 event=event,
-            ).make_info({"execution_time": time.monotonic() - ts})
+            )
         elif self.runs_on.type == SELF_HOSTED:  # pragma: no cov
             pass
         elif self.runs_on.type == AZ_BATCH:  # pragma: no cov
             from .plugins.providers.az import azure_batch_execute
 
-            return azure_batch_execute(
+            rs = azure_batch_execute(
                 self,
                 params,
                 run_id=parent_run_id,
                 event=event,
-            ).make_info({"execution_time": time.monotonic() - ts})
+            )
         elif self.runs_on.type == DOCKER:  # pragma: no cov
-            return docker_execution(
+            rs = docker_process(
                 self,
                 params,
                 run_id=parent_run_id,
                 event=event,
-            ).make_info({"execution_time": time.monotonic() - ts})
+            )
         elif self.runs_on.type == CONTAINER:  # pragma: no cov
             from .plugins.providers.container import container_execute
 
-            return container_execute(
+            rs = container_execute(
                 self,
                 params,
                 run_id=parent_run_id,
                 event=event,
-            ).make_info({"execution_time": time.monotonic() - ts})
+            )
         elif self.runs_on.type == AWS_BATCH:  # pragma: no cov
             from .plugins.providers.aws import aws_batch_execute
 
-            return aws_batch_execute(
+            rs = aws_batch_execute(
                 self,
                 params,
                 run_id=parent_run_id,
                 event=event,
-            ).make_info({"execution_time": time.monotonic() - ts})
+            )
         elif self.runs_on.type == GCP_BATCH:  # pragma: no cov
             from .plugins.providers.gcs import gcp_batch_execute
 
-            return gcp_batch_execute(
+            rs = gcp_batch_execute(
                 self,
                 params,
                 run_id=parent_run_id,
                 event=event,
-            ).make_info({"execution_time": time.monotonic() - ts})
+            )
 
-        trace.error(
-            f"[JOB]: Execution not support runs-on: {self.runs_on.type.value!r} "
-            f"yet."
-        )
-        return Result(
-            status=FAILED,
-            run_id=run_id,
-            parent_run_id=parent_run_id,
-            context={
-                "status": FAILED,
-                "errors": JobError(
-                    f"Execute runs-on type: {self.runs_on.type.value!r} does "
-                    f"not support yet."
-                ).to_dict(),
-            },
-            info={"execution_time": time.monotonic() - ts},
-            extras=self.extras,
+        if rs is None:
+            trace.error(
+                f"[JOB]: Execution not support runs-on: {self.runs_on.type.value!r} "
+                f"yet."
+            )
+            return Result(
+                status=FAILED,
+                run_id=run_id,
+                parent_run_id=parent_run_id,
+                context={
+                    "status": FAILED,
+                    "errors": JobError(
+                        f"Job runs-on type: {self.runs_on.type.value!r} does "
+                        f"not support yet."
+                    ).to_dict(),
+                },
+                extras=self.extras,
+            )
+
+        if rs.status == SKIP:
+            raise JobSkipError("Job got skipped status.")
+        elif rs.status == CANCEL:
+            raise JobCancelError("Job got canceled status.")
+        elif rs.status == FAILED:
+            raise JobError("Job process error")
+        return rs
+
+    def _execute(
+        self,
+        params: DictData,
+        context: DictData,
+        trace: Trace,
+        event: Optional[Event] = None,
+    ) -> Result:
+        """Wrapped the route execute method before returning to handler
+        execution.
+
+            This method call to make retry strategy for process routing
+        method.
+
+        Args:
+            params: A parameter data that want to use in this execution
+            context:
+            trace (Trace):
+            event (Event, default None):
+
+        Returns:
+            Result: The wrapped execution result.
+        """
+        current_retry: int = 0
+        maximum_retry: int = self.retry + 1
+        exception: Exception
+        catch(context, status=WAIT)
+        try:
+            return self.process(
+                params,
+                run_id=trace.run_id,
+                context=context,
+                parent_run_id=trace.parent_run_id,
+                event=event,
+            )
+        except (JobCancelError, JobSkipError):
+            trace.debug("[JOB]: process raise skip or cancel error.")
+            raise
+        except Exception as e:
+            if self.retry == 0:
+                raise
+
+            current_retry += 1
+            exception = e
+
+        trace.warning(
+            f"[JOB]: Retry count: {current_retry}/{maximum_retry} ... "
+            f"( {exception.__class__.__name__} )"
         )
+        while current_retry < maximum_retry:
+            try:
+                catch(
+                    context=context,
+                    status=WAIT,
+                    updated={"retry": current_retry},
+                )
+                return self.process(
+                    params,
+                    run_id=trace.run_id,
+                    context=context,
+                    parent_run_id=trace.parent_run_id,
+                    event=event,
+                )
+            except (JobCancelError, JobSkipError):
+                trace.debug("[JOB]: process raise skip or cancel error.")
+                raise
+            except Exception as e:
+                current_retry += 1
+                trace.warning(
+                    f"[JOB]: Retry count: {current_retry}/{maximum_retry} ... "
+                    f"( {e.__class__.__name__} )"
+                )
+                exception = e
+                time.sleep(1.2**current_retry)
 
+        trace.error(f"[JOB]: Reach the maximum of retry number: {self.retry}.")
+        raise exception
 
-def mark_errors(context: DictData, error: JobError) -> None:
-    """Make the errors context result with the refs value depends on the nested
-    execute func.
+    def execute(
+        self,
+        params: DictData,
+        *,
+        run_id: StrOrNone = None,
+        event: Optional[Event] = None,
+    ) -> Result:
+        """Job execution with passing dynamic parameters from the workflow
+        execution. It will generate matrix values at the first step and run
+        multithread on this metrics to the `stages` field of this job.
 
-    :param context: (DictData) A context data.
-    :param error: (JobError) A stage exception object.
-    """
-    if "errors" in context:
-        context["errors"][error.refs] = error.to_dict()
-    else:
-        context["errors"] = error.to_dict(with_refs=True)
+            This method be execution routing for call dynamic execution function
+        with specific target `runs-on` value.
+
+        Args
+            params: (DictData) A parameter context that also pass from the
+                workflow execute method.
+            run_id: (str) An execution running ID.
+            event: (Event) An Event manager instance that use to cancel this
+                execution if it forces stopped by parent execution.
+
+        Returns
+            Result: Return Result object that create from execution context.
+        """
+        ts: float = time.monotonic()
+        parent_run_id, run_id = extract_id(
+            (self.id or "EMPTY"), run_id=run_id, extras=self.extras
+        )
+        context: DictData = {
+            "status": WAIT,
+            "info": {"exec_start": get_dt_now()},
+        }
+        trace: Trace = get_trace(
+            run_id, parent_run_id=parent_run_id, extras=self.extras
+        )
+        try:
+            trace.info(
+                f"[JOB]: Handler {self.runs_on.type.name}: "
+                f"{(self.id or 'EMPTY')!r}."
+            )
+            result: Result = self._execute(
+                params,
+                context=context,
+                trace=trace,
+                event=event,
+            )
+            return result
+        except JobError as e:  # pragma: no cov
+            if isinstance(e, JobSkipError):
+                trace.error(f"[JOB]: ⏭️ Skip: {e}")
+
+            st: Status = get_status_from_error(e)
+            return Result.from_trace(trace).catch(
+                status=st, context=catch(context, status=st)
+            )
+        finally:
+            context["info"].update(
+                {
+                    "exec_end": get_dt_now(),
+                    "exec_latency": round(time.monotonic() - ts, 6),
+                }
+            )
+            trace.debug("[JOB]: End Handler job execution.")
 
 
 def pop_stages(context: DictData) -> DictData:
+    """Pop a stages key from the context data. It will return empty dict if it
+    does not exist.
+    """
     return filter_func(context.pop("stages", {}))
 
 
-def local_execute_strategy(
+def local_process_strategy(
     job: Job,
     strategy: DictData,
     params: DictData,
-    run_id: str,
+    trace: Trace,
     context: DictData,
     *,
-    parent_run_id: Optional[str] = None,
     event: Optional[Event] = None,
 ) -> tuple[Status, DictData]:
     """Local strategy execution with passing dynamic parameters from the
@@ -1014,25 +1164,24 @@ def local_execute_strategy(
     For each stage that execution with this strategy metrix, it will use the
     `set_outputs` method for reconstruct result context data.
 
-    :param job: (Job) A job model that want to execute.
-    :param strategy: (DictData) A strategy metrix value. This value will pass
-        to the `matrix` key for templating in context data.
-    :param params: (DictData) A parameter data.
-    :param run_id: (str)
-        :param context: (DictData)
-        :param parent_run_id: (str | None)
-    :param event: (Event) An Event manager instance that use to cancel this
-        execution if it forces stopped by parent execution.
-
-    :raise JobError: If event was set.
-    :raise JobError: If stage execution raise any error as `StageError`.
-    :raise JobError: If the result from execution has `FAILED` status.
-
-    :rtype: tuple[Status, DictData]
+    Args:
+        job (Job): A job model that want to execute.
+        strategy (DictData): A strategy metrix value. This value will pass
+            to the `matrix` key for templating in context data.
+        params (DictData): A parameter data.
+        trace (Trace):
+        context (DictData):
+        event (Event): An Event manager instance that use to cancel this
+            execution if it forces stopped by parent execution.
+
+    Raises:
+        JobError: If event was set.
+        JobError: If stage execution raise any error as `StageError`.
+        JobError: If the result from execution has `FAILED` status.
+
+    Returns:
+        tuple[Status, DictData]: A pair of Status and DictData objects.
     """
-    trace: Trace = get_trace(
-        run_id, parent_run_id=parent_run_id, extras=job.extras
-    )
     if strategy:
         strategy_id: str = gen_id(strategy)
         trace.info(f"[JOB]: Execute Strategy: {strategy_id!r}")
@@ -1071,7 +1220,7 @@ def local_execute_strategy(
         trace.info(f"[JOB]: Execute Stage: {stage.iden!r}")
         rs: Result = stage.execute(
             params=current_context,
-            run_id=parent_run_id,
+            run_id=trace.parent_run_id,
             event=event,
         )
         stage.set_outputs(rs.context, to=current_context)
@@ -1133,11 +1282,12 @@ def local_execute_strategy(
     return status, context
 
 
-def local_execute(
+def local_process(
     job: Job,
     params: DictData,
+    run_id: str,
+    context: DictData,
     *,
-    run_id: StrOrNone = None,
     event: Optional[Event] = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
@@ -1155,21 +1305,23 @@ def local_execute(
             ]
         }
 
-    :param job: (Job) A job model.
-    :param params: (DictData) A parameter data.
-    :param run_id: (str) A job running ID.
-    :param event: (Event) An Event manager instance that use to cancel this
-        execution if it forces stopped by parent execution.
+    Args:
+        job (Job): A job model.
+        params (DictData): A parameter data.
+        run_id (str): A job running ID.
+        context (DictData):
+        event (Event, default None): An Event manager instance that use to
+            cancel this execution if it forces stopped by parent execution.
 
-    :rtype: Result
+    Returns:
+        Result: A job process result.
     """
-    ts: float = time.monotonic()
-    parent_run_id: StrOrNone = run_id
-    run_id: str = gen_id((job.id or "EMPTY"), unique=True)
+    parent_run_id, run_id = extract_id(
+        (job.id or "EMPTY"), run_id=run_id, extras=job.extras
+    )
     trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
-    context: DictData = {"status": WAIT}
     trace.info("[JOB]: Start Local executor.")
 
     if job.desc:
@@ -1182,7 +1334,6 @@ def local_execute(
             parent_run_id=parent_run_id,
             status=SKIP,
             context=catch(context, status=SKIP),
-            info={"execution_time": time.monotonic() - ts},
             extras=job.extras,
         )
 
@@ -1208,7 +1359,6 @@ def local_execute(
                     status=FAILED,
                     updated={"errors": to_dict(err)},
                 ),
-                info={"execution_time": time.monotonic() - ts},
                 extras=job.extras,
             )
     if workers >= 10:
@@ -1226,7 +1376,6 @@ def local_execute(
                 status=FAILED,
                 updated={"errors": JobError(err_msg).to_dict()},
             ),
-            info={"execution_time": time.monotonic() - ts},
             extras=job.extras,
         )
 
@@ -1252,20 +1401,18 @@ def local_execute(
                     ).to_dict()
                 },
             ),
-            info={"execution_time": time.monotonic() - ts},
             extras=job.extras,
         )
 
     with ThreadPoolExecutor(workers, "jb_stg") as executor:
         futures: list[Future] = [
             executor.submit(
-                local_execute_strategy,
+                local_process_strategy,
                 job=job,
                 strategy=strategy,
                 params=params,
-                run_id=run_id,
+                trace=trace,
                 context=context,
-                parent_run_id=parent_run_id,
                 event=event,
             )
             for strategy in strategies
@@ -1314,11 +1461,10 @@ def local_execute(
     return Result.from_trace(trace).catch(
         status=status,
         context=catch(context, status=status, updated=errors),
-        info={"execution_time": time.monotonic() - ts},
     )
 
 
-def self_hosted_execute(
+def self_hosted_process(
     job: Job,
     params: DictData,
     *,
@@ -1329,13 +1475,15 @@ def self_hosted_execute(
     workflow execution or itself execution. It will make request to the
     self-hosted host url.
 
-    :param job: (Job) A job model that want to execute.
-    :param params: (DictData) A parameter data.
-    :param run_id: (str) A job running ID.
-    :param event: (Event) An Event manager instance that use to cancel this
-        execution if it forces stopped by parent execution.
+    Args:
+        job (Job): A job model that want to execute.
+        params (DictData): A parameter data.
+        run_id (str): A job running ID.
+        event (Event): An Event manager instance that use to cancel this
+            execution if it forces stopped by parent execution.
 
-    :rtype: Result
+    Returns:
+        Result: A Result object.
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
@@ -1402,11 +1550,7 @@ def self_hosted_execute(
     )
 
 
-# Azure Batch execution is now handled by the Azure Batch provider
-# See src/ddeutil/workflow/plugins/providers/az.py for implementation
-
-
-def docker_execution(
+def docker_process(
     job: Job,
     params: DictData,
     *,