ddeutil-workflow 0.0.53__py3-none-any.whl → 0.0.54__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.53"
+__version__: str = "0.0.54"

ddeutil/workflow/api/api.py

@@ -24,8 +24,8 @@ from ..conf import api_config, config
 from ..logs import get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
-from .repeat import repeat_at
 from .routes import job, log
+from .utils import repeat_at
 
 load_dotenv()
 logger = get_logger("uvicorn.error")

ddeutil/workflow/api/routes/job.py

@@ -39,7 +39,7 @@ async def job_execute(
     job: Job,
     params: dict[str, Any],
 ):
-    """Execute job via API."""
+    """Execute job via RestAPI."""
     rs: Result = Result(
         context=result.context,
         run_id=result.run_id,
@@ -56,7 +56,7 @@ async def job_execute(
             to=context,
         )
     except JobException as err:
-        rs.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
+        rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
 
     return {
         "message": "Start execute job via API.",

ddeutil/workflow/job.py

@@ -3,12 +3,17 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Job Model that use for keeping stages and node that running its stages.
-The job handle the lineage of stages and location of execution of stages that
-mean the job model able to define `runs-on` key that allow you to run this
-job.
+"""Job model that use for store Stage models and node parameter that use for
+running these stages. The job model handle the lineage of stages and location of
+execution that mean you can define `runs-on` field with the Self-Hosted mode
+for execute on target machine instead of the current local machine.
 
-    This module include Strategy Model that use on the job strategy field.
+    This module include Strategy model that use on the job `strategy` field for
+making matrix values before execution parallelism stage execution.
+
+    The Job model does not implement `handler_execute` same as Stage model
+because the job should raise only `JobException` class from the execution
+method.
 """
 from __future__ import annotations
 
@@ -41,7 +46,7 @@ from .exceptions import (
 from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
 from .stages import Stage
-from .utils import cross_product, filter_func, gen_id
+from .utils import NEWLINE, cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -59,10 +64,10 @@ def make(
         This function use the `lru_cache` decorator function increase
     performance for duplicate matrix value scenario.
 
-    :param matrix: A matrix values that want to cross product to possible
-        parallelism values.
-    :param include: A list of additional matrix that want to adds-in.
-    :param exclude: A list of exclude matrix that want to filter-out.
+    :param matrix: (Matrix) A matrix values that want to cross product to
+        possible parallelism values.
+    :param include: (A list of additional matrix that want to adds-in.
+    :param exclude: (A list of exclude matrix that want to filter-out.
 
     :rtype: list[DictStr]
     """
@@ -320,12 +325,13 @@ class Job(BaseModel):
     id: Optional[str] = Field(
         default=None,
         description=(
-            "A job ID that it will add from workflow after validation process."
+            "A job ID that was set from Workflow model after initialize step. "
+            "If this model create standalone, it will be None."
         ),
     )
     desc: Optional[str] = Field(
         default=None,
-        description="A job description that can be string of markdown content.",
+        description="A job description that can be markdown syntax.",
     )
     runs_on: RunsOnModel = Field(
         default_factory=OnLocal,
@@ -339,7 +345,7 @@ class Job(BaseModel):
     )
     stages: list[Stage] = Field(
         default_factory=list,
-        description="A list of Stage of this job.",
+        description="A list of Stage model of this job.",
     )
     trigger_rule: Rule = Field(
         default=Rule.ALL_SUCCESS,
@@ -373,7 +379,7 @@ class Job(BaseModel):
 
     @field_validator("stages", mode="after")
     def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
-        """Validate a stage ID of all stage in stages field should not be
+        """Validate stage ID of each stage in the `stages` field should not be
         duplicate.
 
         :rtype: list[Stage]
@@ -391,11 +397,10 @@ class Job(BaseModel):
 
     @model_validator(mode="after")
     def __validate_job_id__(self) -> Self:
-        """Validate job id should not have templating syntax.
+        """Validate job id should not dynamic with params template.
 
         :rtype: Self
         """
-        # VALIDATE: Validate job id should not dynamic with params template.
         if has_template(self.id):
             raise ValueError(
                 f"Job ID, {self.id!r}, should not has any template."
@@ -419,14 +424,11 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage {stage_id!r} does not exists in this job.")
 
-    def check_needs(
-        self,
-        jobs: dict[str, Any],
-    ) -> Status:  # pragma: no cov
-        """Return Status enum for checking job's need trigger logic in an
-        input list of job's ID.
+    def check_needs(self, jobs: dict[str, Any]) -> Status:  # pragma: no cov
+        """Return trigger status from checking job's need trigger rule logic was
+        valid. The return status should be SUCCESS, FAILED, WAIT, or SKIP.
 
-        :param jobs: A mapping of job ID and result context.
+        :param jobs: A mapping of job ID and its context data.
 
         :raise NotImplementedError: If the job trigger rule out of scope.
 
@@ -441,7 +443,6 @@ class Job(BaseModel):
         need_exist: dict[str, Any] = {
             need: jobs[need] for need in self.needs if need in jobs
         }
-
         if len(need_exist) != len(self.needs):
             return WAIT
         elif all("skipped" in need_exist[job] for job in need_exist):
@@ -469,30 +470,26 @@ class Job(BaseModel):
         elif self.trigger_rule == Rule.NONE_FAILED:
             rs = all("errors" not in need_exist[job] for job in need_exist)
         else:  # pragma: no cov
-            raise NotImplementedError(
-                f"Trigger rule: {self.trigger_rule} does not support yet."
-            )
+            return FAILED
         return make_return(rs)
 
-    def is_skipped(self, params: DictData | None = None) -> bool:
+    def is_skipped(self, params: DictData) -> bool:
         """Return true if condition of this job do not correct. This process
         use build-in eval function to execute the if-condition.
 
+        :param params: (DictData) A parameter value that want to pass to condition
+            template.
+
         :raise JobException: When it has any error raise from the eval
             condition statement.
         :raise JobException: When return type of the eval condition statement
             does not return with boolean type.
 
-        :param params: (DictData) A parameters that want to pass to condition
-            template.
-
         :rtype: bool
         """
         if self.condition is None:
             return False
 
-        params: DictData = {} if params is None else params
-
         try:
             # WARNING: The eval build-in function is very dangerous. So, it
             #   should use the `re` module to validate eval-string before
@@ -513,15 +510,20 @@ class Job(BaseModel):
         output: DictData,
         to: DictData,
         *,
-        job_id: Optional[None] = None,
+        job_id: Optional[str] = None,
     ) -> DictData:
-        """Set an outputs from execution process to the received context. The
-        result from execution will pass to value of `strategies` key.
+        """Set an outputs from execution result context to the received context
+        with a `to` input parameter. The result context from job strategy
+        execution will be set with `strategies` key in this job ID key.
 
             For example of setting output method, If you receive execute output
         and want to set on the `to` like;
 
-            ... (i)   output: {'strategy-01': bar, 'strategy-02': bar}
+            ... (i)   output: {
+                        'strategy-01': 'foo',
+                        'strategy-02': 'bar',
+                        'skipped': True,
+                    }
             ... (ii)  to: {'jobs': {}}
 
         The result of the `to` argument will be;
@@ -530,9 +532,10 @@ class Job(BaseModel):
                         'jobs': {
                             '<job-id>': {
                                 'strategies': {
-                                    'strategy-01': bar,
-                                    'strategy-02': bar,
-                                }
+                                    'strategy-01': 'foo',
+                                    'strategy-02': 'bar',
+                                },
+                                'skipped': True,
                             }
                         }
                     }
@@ -540,9 +543,10 @@ class Job(BaseModel):
         :raise JobException: If the job's ID does not set and the setting
             default job ID flag does not set.
 
-        :param output: An output context.
-        :param to: A context data that want to add output result.
-        :param job_id: A job ID if the id field does not set.
+        :param output: (DictData) A result data context that want to extract
+            and transfer to the `strategies` key in receive context.
+        :param to: (DictData) A received context data.
+        :param job_id: (str | None) A job ID if the `id` field does not set.
 
         :rtype: DictData
         """
@@ -555,7 +559,7 @@ class Job(BaseModel):
             )
 
         _id: str = self.id or job_id
-        output: DictData = copy.deepcopy(output)
+        output: DictData = output.copy()
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
@@ -584,9 +588,7 @@ class Job(BaseModel):
         *,
         run_id: str | None = None,
         parent_run_id: str | None = None,
-        result: Result | None = None,
         event: Event | None = None,
-        raise_error: bool = True,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
@@ -595,22 +597,18 @@ class Job(BaseModel):
             This method be execution routing for call dynamic execution function
         with specific target `runs-on` value.
 
-        :param params: An input parameters that use on job execution.
+        :param params: (DictData) A parameter data.
         :param run_id: (str) A job running ID.
         :param parent_run_id: (str) A parent workflow running ID.
-        :param result: (Result) A result object for keeping context and status
-            data.
-        :param event: (Event) An event manager that pass to the
-            PoolThreadExecutor.
-        :param raise_error: (bool) A flag that all this method raise error to
-            the strategy execution. Default is `True`.
+        :param event: (Event) An Event manager instance that use to cancel this
+            execution if it forces stopped by parent execution.
 
-        :raise NotImplementedError: If the `runs-on` value does not implement.
+        :raise NotImplementedError: If the `runs-on` value does not implement on
+            this execution.
 
         :rtype: Result
         """
         result: Result = Result.construct_with_rs_or_id(
-            result,
             run_id=run_id,
             parent_run_id=parent_run_id,
             id_logic=(self.id or "not-set"),
@@ -627,7 +625,6 @@ class Job(BaseModel):
                 run_id=run_id,
                 parent_run_id=parent_run_id,
                 event=event,
-                raise_error=raise_error,
             )
         elif self.runs_on.type == RunsOn.SELF_HOSTED:  # pragma: no cov
             pass
@@ -638,15 +635,14 @@ class Job(BaseModel):
                 run_id=run_id,
                 parent_run_id=parent_run_id,
                 event=event,
-                raise_error=raise_error,
             )
 
         # pragma: no cov
         result.trace.error(
-            f"[JOB]: Execution not support runs-on: {self.runs_on.type!r} yet."
+            f"[JOB]: Execute not support runs-on: {self.runs_on.type!r} yet."
         )
         raise NotImplementedError(
-            f"Execution runs-on type: {self.runs_on.type} does not support yet."
+            f"Execute runs-on type: {self.runs_on.type} does not support yet."
         )
 
 
@@ -657,7 +653,6 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool = True,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -671,32 +666,33 @@ def local_execute_strategy(
     For each stage that execution with this strategy metrix, it will use the
     `set_outputs` method for reconstruct result context data.
 
-    :raise JobException: If it has any error from `StageException` or
-        `UtilException`.
-
     :param job: (Job) A job model that want to execute.
     :param strategy: A strategy metrix value that use on this execution.
         This value will pass to the `matrix` key for templating.
-    :param params: A dynamic parameters that will deepcopy to the context.
-    :param result: (Result) A result object for keeping context and status
-        data.
-    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
-    :param raise_error: (bool) A flag that all this method raise error
+    :param params: (DictData) A parameter data.
+    :param result: (Result) A Result instance for return context and status.
+    :param event: (Event) An Event manager instance that use to cancel this
+        execution if it forces stopped by parent execution.
+
+    :raise JobException: If it has any error from `StageException` or
+        `UtilException`.
 
     :rtype: Result
     """
-    if result is None:
-        result: Result = Result(run_id=gen_id(job.id or "not-set", unique=True))
+    result: Result = result or Result(
+        run_id=gen_id(job.id or "not-set", unique=True),
+        extras=job.extras,
+    )
 
     strategy_id: str = gen_id(strategy)
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
 
     if strategy:
-        result.trace.info(f"[JOB]: Execute Strategy: {strategy_id!r}")
+        result.trace.info(f"[JOB]: Start Strategy: {strategy_id!r}")
         result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
     else:
-        result.trace.info("[JOB]: Execute Empty-Strategy")
+        result.trace.info("[JOB]: Start Strategy: EMPTY")
 
     for stage in job.stages:
 
@@ -735,12 +731,7 @@ def local_execute_strategy(
             stage.set_outputs(rs.context, to=context)
         except (StageException, UtilException) as e:
             result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
-            if raise_error:
-                raise JobException(
-                    f"Stage execution error: {e.__class__.__name__}: {e}"
-                ) from None
-
-            return result.catch(
+            result.catch(
                 status=FAILED,
                 context={
                     strategy_id: {
@@ -750,13 +741,17 @@ def local_execute_strategy(
                     },
                 },
             )
+            raise JobException(
+                f"Stage raise: {e.__class__.__name__}: {e}"
+            ) from e
 
         if rs.status == FAILED:
             error_msg: str = (
-                f"Job strategy was break because stage, {stage.iden}, "
-                f"failed without raise error."
+                f"Strategy break because stage, {stage.iden!r}, return FAILED "
+                f"status."
             )
-            return result.catch(
+            result.trace.warning(f"[JOB]: {error_msg}")
+            result.catch(
                 status=FAILED,
                 context={
                     strategy_id: {
@@ -766,6 +761,7 @@ def local_execute_strategy(
                     },
                 },
             )
+            raise JobException(error_msg)
 
     return result.catch(
         status=SUCCESS,
@@ -785,22 +781,20 @@ def local_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-    raise_error: bool = True,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution or itself execution. It will generate matrix values at the first
     step and run multithread on this metrics to the `stages` field of this job.
 
-        This method does not raise any JobException if it runs with
+        This method does not raise any `JobException` if it runs with
     multi-threading strategy.
 
-    :param job: (Job) A job model that want to execute.
-    :param params: (DictData) An input parameters that use on job execution.
-    :param run_id: (str) A job running ID for this execution.
-    :param parent_run_id: (str) A parent workflow running ID for this release.
-    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
-    :param raise_error: (bool) A flag that all this method raise error to the
-        strategy execution. Default is `True`.
+    :param job: (Job) A job model.
+    :param params: (DictData) A parameter data.
+    :param run_id: (str) A job running ID.
+    :param parent_run_id: (str) A parent workflow running ID.
+    :param event: (Event) An Event manager instance that use to cancel this
+        execution if it forces stopped by parent execution.
 
     :rtype: Result
     """
@@ -812,40 +806,12 @@ def local_execute(
     )
 
     event: Event = Event() if event is None else event
-
-    # NOTE: Normal Job execution without parallel strategy matrix. It uses
-    #   for-loop to control strategy execution sequentially.
-    if (not job.strategy.is_set()) or job.strategy.max_parallel == 1:
-
-        for strategy in job.strategy.make():
-
-            if event and event.is_set():  # pragma: no cov
-                return result.catch(
-                    status=CANCEL,
-                    context={
-                        "errors": JobException(
-                            "Job strategy was canceled from event that had set "
-                            "before strategy execution."
-                        ).to_dict()
-                    },
-                )
-
-            local_execute_strategy(
-                job,
-                strategy,
-                params,
-                result=result,
-                event=event,
-                raise_error=raise_error,
-            )
-
-        return result
-
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
+    workers: int = job.strategy.max_parallel
     result.trace.info(
-        f"[JOB]: Start multithreading: {job.strategy.max_parallel} threads "
-        f"with {ls} mode."
+        f"[JOB]: {ls}-Execute: {job.id} with {workers} "
+        f"worker{'s' if workers > 1 else ''}."
     )
 
     if event and event.is_set():  # pragma: no cov
@@ -853,15 +819,14 @@ def local_execute(
             status=CANCEL,
             context={
                 "errors": JobException(
-                    "Job strategy was canceled from event that had set "
-                    "before strategy execution."
+                    "Job was canceled from event that had set before "
+                    "local execution."
                 ).to_dict()
             },
         )
 
     with ThreadPoolExecutor(
-        max_workers=job.strategy.max_parallel,
-        thread_name_prefix="job_strategy_exec_",
+        max_workers=workers, thread_name_prefix="job_strategy_exec_"
     ) as executor:
 
         futures: list[Future] = [
@@ -872,7 +837,6 @@ def local_execute(
                 params=params,
                 result=result,
                 event=event,
-                raise_error=raise_error,
             )
             for strategy in job.strategy.make()
         ]
@@ -881,12 +845,9 @@ def local_execute(
         status: Status = SUCCESS
 
         if not fail_fast_flag:
-            done = as_completed(futures, timeout=1800)
+            done: list[Future] = as_completed(futures)
         else:
-            done, not_done = wait(
-                futures, timeout=1800, return_when=FIRST_EXCEPTION
-            )
-
+            done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
             if len(done) != len(futures):
                 result.trace.warning(
                     "[JOB]: Set event for stop pending stage future."
@@ -895,10 +856,8 @@ def local_execute(
                 for future in not_done:
                     future.cancel()
 
-            nd: str = (
-                f", the strategies do not run is {not_done}" if not_done else ""
-            )
-            result.trace.debug(f"[JOB]: Strategy set Fail-Fast{nd}")
+            nd: str = f", strategies not run: {not_done}" if not_done else ""
+            result.trace.debug(f"... Strategy set Fail-Fast{nd}")
 
         for future in done:
             try:
@@ -906,10 +865,12 @@ def local_execute(
             except JobException as e:
                 status = FAILED
                 result.trace.error(
-                    f"[JOB]: {ls} Catch:\n\t{e.__class__.__name__}:\n\t{e}"
+                    f"[JOB]: {ls}: {e.__class__.__name__}:{NEWLINE}{e}"
                 )
-                context.update({"errors": e.to_dict()})
-
+                if "errors" in context:
+                    context["errors"].append(e.to_dict())
+                else:
+                    context["errors"] = [e.to_dict()]
     return result.catch(status=status, context=context)
 
 
@@ -920,19 +881,17 @@ def self_hosted_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-    raise_error: bool = True,
 ) -> Result:  # pragma: no cov
     """Self-Hosted job execution with passing dynamic parameters from the
     workflow execution or itself execution. It will make request to the
     self-hosted host url.
 
     :param job: (Job) A job model that want to execute.
-    :param params: (DictData) An input parameters that use on job execution.
-    :param run_id: (str) A job running ID for this execution.
-    :param parent_run_id: (str) A parent workflow running ID for this release.
-    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
-    :param raise_error: (bool) A flag that all this method raise error to the
-        strategy execution.
+    :param params: (DictData) A parameter data.
+    :param run_id: (str) A job running ID.
+    :param parent_run_id: (str) A parent workflow running ID.
+    :param event: (Event) An Event manager instance that use to cancel this
+        execution if it forces stopped by parent execution.
 
     :rtype: Result
     """
@@ -948,8 +907,8 @@ def self_hosted_execute(
             status=CANCEL,
             context={
                 "errors": JobException(
-                    "Job self-hosted execution was canceled from event that "
-                    "had set before start execution."
+                    "Job was canceled from event that had set before start "
+                    "self-hosted execution."
                 ).to_dict()
             },
         )
@@ -964,20 +923,17 @@ def self_hosted_execute(
                 "job": job.model_dump(),
                 "params": params,
                 "result": result.__dict__,
-                "raise_error": raise_error,
             },
         )
     except requests.exceptions.RequestException as e:
         return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
     if resp.status_code != 200:
-        if raise_error:
-            raise JobException(
-                f"Job execution error from request to self-hosted: "
-                f"{job.runs_on.args.host!r}"
-            )
+        raise JobException(
+            f"Job execution error from request to self-hosted: "
+            f"{job.runs_on.args.host!r}"
+        )
 
-        return result.catch(status=FAILED)
     return result.catch(status=SUCCESS)
 
 
@@ -988,7 +944,6 @@ def azure_batch_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
 ) -> Result:  # pragma no cov
     """Azure Batch job execution that will run all job's stages on the Azure
     Batch Node and extract the result file to be returning context result.
@@ -1013,7 +968,6 @@ def azure_batch_execute(
     :param run_id:
     :param parent_run_id:
     :param event:
-    :param raise_error:
 
     :rtype: Result
     """
@@ -1028,13 +982,12 @@ def azure_batch_execute(
             status=CANCEL,
             context={
                 "errors": JobException(
-                    "Job azure-batch execution was canceled from event that "
-                    "had set before start execution."
+                    "Job was canceled from event that had set before start "
+                    "azure-batch execution."
                 ).to_dict()
             },
         )
     print(params)
-    print(raise_error)
     return result.catch(status=SUCCESS)
 
 
@@ -1045,7 +998,6 @@ def docker_execution(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-    raise_error: bool | None = None,
 ):
     """Docker job execution.
 
@@ -1071,5 +1023,4 @@ def docker_execution(
             },
         )
     print(params)
-    print(raise_error)
     return result.catch(status=SUCCESS)