ddeutil-workflow 0.0.20__tar.gz → 0.0.21__tar.gz

{ddeutil_workflow-0.0.20/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.20
+Version: 0.0.21
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -195,6 +195,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_JOB_RAISE_ERROR`         | Core      | true                             | A flag that all job raise JobException from job strategy execution.                                                |        |
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
+| `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

{ddeutil_workflow-0.0.20 → ddeutil_workflow-0.0.21}/README.md

@@ -162,6 +162,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_JOB_RAISE_ERROR`         | Core      | true                             | A flag that all job raise JobException from job strategy execution.                                                |        |
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
+| `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

ddeutil_workflow-0.0.21/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.21"

{ddeutil_workflow-0.0.20 → ddeutil_workflow-0.0.21}/src/ddeutil/workflow/__types.py

@@ -101,6 +101,7 @@ class Re:
     #
     #   Examples:
     #       - tasks/function@dummy
+    #
     __re_task_fmt: str = r"""
         ^
             (?P<path>[^/@]+)

{ddeutil_workflow-0.0.20 → ddeutil_workflow-0.0.21}/src/ddeutil/workflow/conf.py

@@ -100,6 +100,9 @@ class Config:
 
     # NOTE: Workflow
     max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
+    max_job_exec_timeout: int = int(
+        env("WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT", "600")
+    )
     max_poking_pool_worker: int = int(
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )

{ddeutil_workflow-0.0.20 → ddeutil_workflow-0.0.21}/src/ddeutil/workflow/job.py

@@ -11,7 +11,6 @@ job.
 from __future__ import annotations
 
 import copy
-import time
 from concurrent.futures import (
     FIRST_EXCEPTION,
     Future,
@@ -48,13 +47,13 @@ from .utils import (
 )
 
 logger = get_logger("ddeutil.workflow")
-MatrixInclude = list[dict[str, Union[str, int]]]
-MatrixExclude = list[dict[str, Union[str, int]]]
+MatrixFilter = list[dict[str, Union[str, int]]]
 
 
 __all__: TupleStr = (
     "Strategy",
     "Job",
+    "TriggerRules",
     "make",
 )
 
@@ -63,8 +62,8 @@ __all__: TupleStr = (
 @lru_cache
 def make(
     matrix: Matrix,
-    include: MatrixInclude,
-    exclude: MatrixExclude,
+    include: MatrixFilter,
+    exclude: MatrixFilter,
 ) -> list[DictStr]:
     """Make a list of product of matrix values that already filter with
     exclude matrix and add specific matrix with include.
@@ -124,7 +123,7 @@ def make(
 
 
 class Strategy(BaseModel):
-    """Strategy Model that will combine a matrix together for running the
+    """Strategy model that will combine a matrix together for running the
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
@@ -166,11 +165,11 @@ class Strategy(BaseModel):
             "A matrix values that want to cross product to possible strategies."
         ),
     )
-    include: MatrixInclude = Field(
+    include: MatrixFilter = Field(
         default_factory=list,
         description="A list of additional matrix that want to adds-in.",
     )
-    exclude: MatrixExclude = Field(
+    exclude: MatrixFilter = Field(
         default_factory=list,
         description="A list of exclude matrix that want to filter-out.",
     )
@@ -204,7 +203,7 @@ class Strategy(BaseModel):
 
 
 class TriggerRules(str, Enum):
-    """Trigger Rules enum object."""
+    """Trigger rules enum object."""
 
     all_success: str = "all_success"
     all_failed: str = "all_failed"
@@ -215,8 +214,15 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
+class RunsOn(str, Enum):
+    """Runs-On enum object."""
+
+    local: str = "local"
+    docker: str = "docker"
+
+
 class Job(BaseModel):
-    """Job Pydantic model object (group of stages).
+    """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
     you pass matrix mapping and it able to generate, you will see it running
@@ -327,7 +333,10 @@ class Job(BaseModel):
         return self
 
     def stage(self, stage_id: str) -> Stage:
-        """Return stage model that match with an input stage ID.
+        """Return stage instance that exists in this job via passing an input
+        stage ID.
+
+        :raise ValueError: If an input stage ID does not found on this job.
 
         :param stage_id: A stage ID that want to extract from this job.
         :rtype: Stage
@@ -360,8 +369,12 @@ class Job(BaseModel):
                         }
                     }
 
+        :raise JobException: If the job's ID does not set and the setting
+            default job ID flag does not set.
+
         :param output: An output context.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         if self.id is None and not config.job_default_id:
@@ -387,8 +400,8 @@ class Job(BaseModel):
         self,
         strategy: DictData,
         params: DictData,
-        run_id: str | None = None,
         *,
+        run_id: str | None = None,
         event: Event | None = None,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
@@ -398,11 +411,15 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
+            The result of this execution will return result with strategy ID
+        that generated from the `gen_id` function with a input strategy value.
+
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
 
-        :param strategy: A metrix strategy value.
-        :param params: A dynamic parameters.
+        :param strategy: A strategy metrix value that use on this execution.
+            This value will pass to the `matrix` key for templating.
+        :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
         :param event: An manger event that pass to the PoolThreadExecutor.
 
@@ -433,17 +450,19 @@ class Job(BaseModel):
                 logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
                 continue
 
-            logger.info(
-                f"({run_id}) [JOB]: Start execute the stage: {stage.iden!r}"
-            )
+            logger.info(f"({run_id}) [JOB]: Execute stage: {stage.iden!r}")
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({run_id}) [JOB]: Matrix: {strategy}")
+                logger.info(f"({run_id}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
             if event and event.is_set():
+                error_msg: str = (
+                    "Job strategy was canceled from event that had set before "
+                    "strategy execution."
+                )
                 return Result(
                     status=1,
                     context={
@@ -453,15 +472,10 @@ class Job(BaseModel):
                             #   it will not filter function object from context.
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
+                            #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
-                            "error_message": (
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
+                            "error": JobException(error_msg),
+                            "error_message": error_msg,
                         },
                     },
                     run_id=run_id,
@@ -512,7 +526,7 @@ class Job(BaseModel):
                     run_id=run_id,
                 )
 
-            # NOTE: Remove the current stage object.
+            # NOTE: Remove the current stage object for saving memory.
             del stage
 
         return Result(
@@ -542,7 +556,8 @@ class Job(BaseModel):
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy.
+        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
@@ -566,6 +581,7 @@ class Job(BaseModel):
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
         ) as executor:
+
             futures: list[Future] = [
                 executor.submit(
                     self.execute_strategy,
@@ -577,11 +593,8 @@ class Job(BaseModel):
                 for strategy in self.strategy.make()
             ]
 
-            # NOTE: Dynamic catching futures object with fail-fast flag.
             return (
-                self.__catch_fail_fast(
-                    event=event, futures=futures, run_id=run_id
-                )
+                self.__catch_fail_fast(event, futures=futures, run_id=run_id)
                 if self.strategy.fail_fast
                 else self.__catch_all_completed(futures=futures, run_id=run_id)
             )
@@ -593,19 +606,17 @@ class Job(BaseModel):
         run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop all not done futures if it receive the first exception from all
-        running futures.
+        stop and set event on all not done futures if it receive the first
+        exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the
-            observing thread/process.
+            observing multithreading.
         :param futures: A list of futures.
         :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
         rs_final: Result = Result()
@@ -615,9 +626,7 @@ class Job(BaseModel):
         # NOTE: Get results from a collection of tasks with a timeout that has
         #   the first exception.
         done, not_done = wait(
-            futures,
-            timeout=timeout,
-            return_when=FIRST_EXCEPTION,
+            futures, timeout=timeout, return_when=FIRST_EXCEPTION
         )
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
@@ -635,11 +644,13 @@ class Job(BaseModel):
 
         future: Future
         for future in done:
+
+            # NOTE: Handle the first exception from feature
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({run_id}) [JOB]: One stage failed with: "
-                    f"{future.exception()}, shutting down this future."
+                    f"({run_id}) [JOB]: Fail-fast catching:\n\t"
+                    f"{future.exception()}"
                 )
                 context.update(
                     {
@@ -650,7 +661,7 @@ class Job(BaseModel):
                 continue
 
             # NOTE: Update the result context to main job context.
-            context.update(future.result(timeout=result_timeout).context)
+            context.update(future.result().context)
 
         return rs_final.catch(status=status, context=context)
 
@@ -660,45 +671,27 @@ class Job(BaseModel):
         run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with all-completed mode.
 
-        :param futures: A list of futures that want to catch all completed
-            result.
+        :param futures: A list of futures.
         :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
         rs_final: Result = Result()
         context: DictData = {}
         status: int = 0
+
         for future in as_completed(futures, timeout=timeout):
             try:
-                context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:  # pragma: no cov
-                status = 1
-                logger.warning(
-                    f"({run_id}) [JOB]: Task is hanging. Attempting to "
-                    f"kill."
-                )
-                future.cancel()
-                time.sleep(0.1)
-
-                stmt: str = (
-                    "Failed to cancel the task."
-                    if not future.cancelled()
-                    else "Task canceled successfully."
-                )
-                logger.warning(f"({run_id}) [JOB]: {stmt}")
+                context.update(future.result().context)
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({run_id}) [JOB]: Get stage exception with "
-                    f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
-                    f"{err}"
+                    f"({run_id}) [JOB]: All-completed catching:\n\t"
+                    f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(
                     {
@@ -706,4 +699,5 @@ class Job(BaseModel):
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
                 )
+
         return rs_final.catch(status=status, context=context)

{ddeutil_workflow-0.0.20 → ddeutil_workflow-0.0.21}/src/ddeutil/workflow/stage.py

@@ -496,6 +496,7 @@ class PyStage(BaseStage):
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         # NOTE: The output will fileter unnecessary keys from locals.
@@ -638,8 +639,7 @@ class HookStage(BaseStage):
 
         :rtype: Result
         """
-        t_func_hook: str = param2template(self.uses, params)
-        t_func: TagFunc = extract_hook(t_func_hook)()
+        t_func: TagFunc = extract_hook(param2template(self.uses, params))()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.