ddeutil-workflow 0.0.20__py3-none-any.whl → 0.0.21__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.20"
+__version__: str = "0.0.21"

ddeutil/workflow/__types.py

@@ -101,6 +101,7 @@ class Re:
     #
     #   Examples:
     #       - tasks/function@dummy
+    #
     __re_task_fmt: str = r"""
         ^
             (?P<path>[^/@]+)

ddeutil/workflow/conf.py

@@ -100,6 +100,9 @@ class Config:
 
     # NOTE: Workflow
     max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
+    max_job_exec_timeout: int = int(
+        env("WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT", "600")
+    )
     max_poking_pool_worker: int = int(
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )

ddeutil/workflow/job.py

@@ -11,7 +11,6 @@ job.
 from __future__ import annotations
 
 import copy
-import time
 from concurrent.futures import (
     FIRST_EXCEPTION,
     Future,
@@ -48,13 +47,13 @@ from .utils import (
 )
 
 logger = get_logger("ddeutil.workflow")
-MatrixInclude = list[dict[str, Union[str, int]]]
-MatrixExclude = list[dict[str, Union[str, int]]]
+MatrixFilter = list[dict[str, Union[str, int]]]
 
 
 __all__: TupleStr = (
     "Strategy",
     "Job",
+    "TriggerRules",
     "make",
 )
 
@@ -63,8 +62,8 @@ __all__: TupleStr = (
 @lru_cache
 def make(
     matrix: Matrix,
-    include: MatrixInclude,
-    exclude: MatrixExclude,
+    include: MatrixFilter,
+    exclude: MatrixFilter,
 ) -> list[DictStr]:
     """Make a list of product of matrix values that already filter with
     exclude matrix and add specific matrix with include.
@@ -124,7 +123,7 @@ def make(
 
 
 class Strategy(BaseModel):
-    """Strategy Model that will combine a matrix together for running the
+    """Strategy model that will combine a matrix together for running the
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
@@ -166,11 +165,11 @@ class Strategy(BaseModel):
             "A matrix values that want to cross product to possible strategies."
         ),
     )
-    include: MatrixInclude = Field(
+    include: MatrixFilter = Field(
         default_factory=list,
         description="A list of additional matrix that want to adds-in.",
     )
-    exclude: MatrixExclude = Field(
+    exclude: MatrixFilter = Field(
         default_factory=list,
         description="A list of exclude matrix that want to filter-out.",
     )
@@ -204,7 +203,7 @@ class Strategy(BaseModel):
 
 
 class TriggerRules(str, Enum):
-    """Trigger Rules enum object."""
+    """Trigger rules enum object."""
 
     all_success: str = "all_success"
     all_failed: str = "all_failed"
@@ -215,8 +214,15 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
+class RunsOn(str, Enum):
+    """Runs-On enum object."""
+
+    local: str = "local"
+    docker: str = "docker"
+
+
 class Job(BaseModel):
-    """Job Pydantic model object (group of stages).
+    """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
     you pass matrix mapping and it able to generate, you will see it running
@@ -327,7 +333,10 @@ class Job(BaseModel):
         return self
 
     def stage(self, stage_id: str) -> Stage:
-        """Return stage model that match with an input stage ID.
+        """Return stage instance that exists in this job via passing an input
+        stage ID.
+
+        :raise ValueError: If an input stage ID does not found on this job.
 
         :param stage_id: A stage ID that want to extract from this job.
         :rtype: Stage
@@ -360,8 +369,12 @@ class Job(BaseModel):
                         }
                     }
 
+        :raise JobException: If the job's ID does not set and the setting
+            default job ID flag does not set.
+
         :param output: An output context.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         if self.id is None and not config.job_default_id:
@@ -387,8 +400,8 @@ class Job(BaseModel):
         self,
         strategy: DictData,
         params: DictData,
-        run_id: str | None = None,
         *,
+        run_id: str | None = None,
         event: Event | None = None,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
@@ -398,11 +411,15 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
+            The result of this execution will return result with strategy ID
+        that generated from the `gen_id` function with a input strategy value.
+
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
 
-        :param strategy: A metrix strategy value.
-        :param params: A dynamic parameters.
+        :param strategy: A strategy metrix value that use on this execution.
+            This value will pass to the `matrix` key for templating.
+        :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
         :param event: An manger event that pass to the PoolThreadExecutor.
 
@@ -433,17 +450,19 @@ class Job(BaseModel):
                 logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
                 continue
 
-            logger.info(
-                f"({run_id}) [JOB]: Start execute the stage: {stage.iden!r}"
-            )
+            logger.info(f"({run_id}) [JOB]: Execute stage: {stage.iden!r}")
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({run_id}) [JOB]: Matrix: {strategy}")
+                logger.info(f"({run_id}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
             if event and event.is_set():
+                error_msg: str = (
+                    "Job strategy was canceled from event that had set before "
+                    "strategy execution."
+                )
                 return Result(
                     status=1,
                     context={
@@ -453,15 +472,10 @@ class Job(BaseModel):
                             #   it will not filter function object from context.
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
+                            #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
-                            "error_message": (
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
+                            "error": JobException(error_msg),
+                            "error_message": error_msg,
                         },
                     },
                     run_id=run_id,
@@ -512,7 +526,7 @@ class Job(BaseModel):
                     run_id=run_id,
                 )
 
-            # NOTE: Remove the current stage object.
+            # NOTE: Remove the current stage object for saving memory.
             del stage
 
         return Result(
@@ -542,7 +556,8 @@ class Job(BaseModel):
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy.
+        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
@@ -566,6 +581,7 @@ class Job(BaseModel):
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
         ) as executor:
+
             futures: list[Future] = [
                 executor.submit(
                     self.execute_strategy,
@@ -577,11 +593,8 @@ class Job(BaseModel):
                 for strategy in self.strategy.make()
             ]
 
-            # NOTE: Dynamic catching futures object with fail-fast flag.
             return (
-                self.__catch_fail_fast(
-                    event=event, futures=futures, run_id=run_id
-                )
+                self.__catch_fail_fast(event, futures=futures, run_id=run_id)
                 if self.strategy.fail_fast
                 else self.__catch_all_completed(futures=futures, run_id=run_id)
             )
@@ -593,19 +606,17 @@ class Job(BaseModel):
         run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop all not done futures if it receive the first exception from all
-        running futures.
+        stop and set event on all not done futures if it receive the first
+        exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the
-            observing thread/process.
+            observing multithreading.
         :param futures: A list of futures.
         :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
         rs_final: Result = Result()
@@ -615,9 +626,7 @@ class Job(BaseModel):
         # NOTE: Get results from a collection of tasks with a timeout that has
         #   the first exception.
         done, not_done = wait(
-            futures,
-            timeout=timeout,
-            return_when=FIRST_EXCEPTION,
+            futures, timeout=timeout, return_when=FIRST_EXCEPTION
         )
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
@@ -635,11 +644,13 @@ class Job(BaseModel):
 
         future: Future
         for future in done:
+
+            # NOTE: Handle the first exception from feature
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({run_id}) [JOB]: One stage failed with: "
-                    f"{future.exception()}, shutting down this future."
+                    f"({run_id}) [JOB]: Fail-fast catching:\n\t"
+                    f"{future.exception()}"
                 )
                 context.update(
                     {
@@ -650,7 +661,7 @@ class Job(BaseModel):
                 continue
 
             # NOTE: Update the result context to main job context.
-            context.update(future.result(timeout=result_timeout).context)
+            context.update(future.result().context)
 
         return rs_final.catch(status=status, context=context)
 
@@ -660,45 +671,27 @@ class Job(BaseModel):
         run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with all-completed mode.
 
-        :param futures: A list of futures that want to catch all completed
-            result.
+        :param futures: A list of futures.
         :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
         rs_final: Result = Result()
         context: DictData = {}
         status: int = 0
+
         for future in as_completed(futures, timeout=timeout):
             try:
-                context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:  # pragma: no cov
-                status = 1
-                logger.warning(
-                    f"({run_id}) [JOB]: Task is hanging. Attempting to "
-                    f"kill."
-                )
-                future.cancel()
-                time.sleep(0.1)
-
-                stmt: str = (
-                    "Failed to cancel the task."
-                    if not future.cancelled()
-                    else "Task canceled successfully."
-                )
-                logger.warning(f"({run_id}) [JOB]: {stmt}")
+                context.update(future.result().context)
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({run_id}) [JOB]: Get stage exception with "
-                    f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
-                    f"{err}"
+                    f"({run_id}) [JOB]: All-completed catching:\n\t"
+                    f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(
                     {
@@ -706,4 +699,5 @@ class Job(BaseModel):
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
                 )
+
         return rs_final.catch(status=status, context=context)

ddeutil/workflow/stage.py

@@ -496,6 +496,7 @@ class PyStage(BaseStage):
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         # NOTE: The output will fileter unnecessary keys from locals.
@@ -638,8 +639,7 @@ class HookStage(BaseStage):
 
         :rtype: Result
         """
-        t_func_hook: str = param2template(self.uses, params)
-        t_func: TagFunc = extract_hook(t_func_hook)()
+        t_func: TagFunc = extract_hook(param2template(self.uses, params))()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.

ddeutil/workflow/workflow.py

@@ -3,8 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""
-The main schedule running is ``workflow_runner`` function that trigger the
+"""The main schedule running is ``workflow_runner`` function that trigger the
 multiprocess of ``workflow_control`` function for listing schedules on the
 config by ``Loader.finds(Schedule)``.
 
@@ -12,6 +11,7 @@ config by ``Loader.finds(Schedule)``.
 functions; ``workflow_task``, and ``workflow_monitor``.
 
     ``workflow_control`` --- Every minute at :02 --> ``workflow_task``
+
                          --- Every 5 minutes     --> ``workflow_monitor``
 
     The ``workflow_task`` will run ``task.release`` method in threading object
@@ -70,7 +70,7 @@ __all__: TupleStr = (
 @total_ordering
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class WorkflowRelease:
-    """Workflow release data dataclass object."""
+    """Workflow release Pydantic dataclass object."""
 
     date: datetime
     offset: float
@@ -86,6 +86,12 @@ class WorkflowRelease:
 
     @classmethod
     def from_dt(cls, dt: datetime) -> Self:
+        """Construct WorkflowRelease via datetime object only.
+
+        :param dt: A datetime object.
+
+        :rtype: Self
+        """
         return cls(
             date=dt,
             offset=0,
@@ -95,6 +101,9 @@ class WorkflowRelease:
         )
 
     def __eq__(self, other: WorkflowRelease | datetime) -> bool:
+        """Override equal property that will compare only the same type or
+        datetime.
+        """
         if isinstance(other, self.__class__):
             return self.date == other.date
         elif isinstance(other, datetime):
@@ -102,6 +111,9 @@ class WorkflowRelease:
         return NotImplemented
 
     def __lt__(self, other: WorkflowRelease | datetime) -> bool:
+        """Override equal property that will compare only the same type or
+        datetime.
+        """
         if isinstance(other, self.__class__):
             return self.date < other.date
         elif isinstance(other, datetime):
@@ -119,14 +131,19 @@ class WorkflowQueue:
 
     @property
     def is_queued(self) -> bool:
-        """Return True if it has data in the queue."""
+        """Return True if it has workflow release object in the queue.
+
+        :rtype: bool
+        """
         return len(self.queue) > 0
 
     def check_queue(self, data: WorkflowRelease) -> bool:
         """Check a WorkflowRelease value already exists in list of tracking
         queues.
 
-        :param data:
+        :param data: A workflow release object.
+
+        :rtype: bool
         """
         return (
             (data in self.queue)
@@ -135,23 +152,28 @@ class WorkflowQueue:
         )
 
     def push_queue(self, data: WorkflowRelease) -> Self:
+        """Push data to the queue."""
         heappush(self.queue, data)
         return self
 
     def push_running(self, data: WorkflowRelease) -> Self:
+        """Push data to the running."""
         heappush(self.running, data)
         return self
 
     def remove_running(self, data: WorkflowRelease) -> Self:
+        """Remove data on the running if it exists."""
         if data in self.running:
             self.running.remove(data)
 
 
 class Workflow(BaseModel):
-    """Workflow Pydantic Model this is the main future of this project because
-    it use to be workflow data for running everywhere that you want or using it
-    to scheduler task in background. It use lightweight coding line from
-    Pydantic Model and enhance execute method on it.
+    """Workflow Pydantic model.
+
+        This is the main future of this project because it use to be workflow
+    data for running everywhere that you want or using it to scheduler task in
+    background. It use lightweight coding line from Pydantic Model and enhance
+    execute method on it.
     """
 
     name: str = Field(description="A workflow name.")
@@ -184,9 +206,12 @@ class Workflow(BaseModel):
         an input workflow name. The loader object will use this workflow name to
         searching configuration data of this workflow model in conf path.
 
+        :raise ValueError: If the type does not match with current object.
+
         :param name: A workflow name that want to pass to Loader object.
         :param externals: An external parameters that want to pass to Loader
             object.
+
         :rtype: Self
         """
         loader: Loader = Loader(name, externals=(externals or {}))
@@ -235,7 +260,7 @@ class Workflow(BaseModel):
 
     @model_validator(mode="before")
     def __prepare_model_before__(cls, values: DictData) -> DictData:
-        """Prepare the params key."""
+        """Prepare the params key in the data model before validating."""
         # NOTE: Prepare params type if it passing with only type value.
         if params := values.pop("params", {}):
             values["params"] = {
@@ -260,7 +285,15 @@ class Workflow(BaseModel):
     @field_validator("on", mode="after")
     def __on_no_dup__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
-        contain every minute value, it should has only one on value."""
+        contain the every minute value more than one value, it will remove to
+        only one value.
+
+        :raise ValueError: If it has some duplicate value.
+
+        :param value: A list of on object.
+
+        :rtype: list[On]
+        """
         set_ons: set[str] = {str(on.cronjob) for on in value}
         if len(set_ons) != len(value):
             raise ValueError(
@@ -279,6 +312,9 @@ class Workflow(BaseModel):
     def __validate_jobs_need__(self) -> Self:
         """Validate each need job in any jobs should exists.
 
+        :raise WorkflowException: If it has not exists need value in this
+            workflow job.
+
         :rtype: Self
         """
         for job in self.jobs:
@@ -623,8 +659,7 @@ class Workflow(BaseModel):
             # WARNING: This poking method does not allow to use fail-fast
             #   logic to catching parallel execution result.
             for future in as_completed(futures):
-                rs: Result = future.result(timeout=60)
-                results.append(rs.set_parent_run_id(run_id))
+                results.append(future.result().set_parent_run_id(run_id))
 
         while len(workflow_queue.running) > 0:  # pragma: no cov
             logger.warning(
@@ -639,17 +674,19 @@ class Workflow(BaseModel):
         self,
         job_id: str,
         params: DictData,
-        run_id: str | None = None,
         *,
+        run_id: str | None = None,
         raise_error: bool = True,
     ) -> Result:
-        """Workflow Job execution with passing dynamic parameters from the
-        workflow execution to the target job.
+        """Job execution with passing dynamic parameters from the main workflow
+        execution to the target job object via job's ID.
 
             This execution is the minimum level of execution of this workflow
         model. It different with ``self.execute`` because this method run only
         one job and return with context of this job data.
 
+        :raise NotImplementedError: If set raise_error argument to False.
+
         :param job_id: A job ID that want to execute.
         :param params: A params that was parameterized from workflow execution.
         :param run_id: A workflow running ID for this job execution.
@@ -663,11 +700,11 @@ class Workflow(BaseModel):
         # VALIDATE: check a job ID that exists in this workflow or not.
         if job_id not in self.jobs:
             raise WorkflowException(
-                f"The job ID: {job_id} does not exists in {self.name!r} "
+                f"The job: {job_id!r} does not exists in {self.name!r} "
                 f"workflow."
             )
 
-        logger.info(f"({run_id}) [WORKFLOW]: Start execute: {job_id!r}")
+        logger.info(f"({run_id}) [WORKFLOW]: Start execute job: {job_id!r}")
 
         # IMPORTANT:
         #   Change any job running IDs to this workflow running ID.
@@ -686,17 +723,18 @@ class Workflow(BaseModel):
                 raise WorkflowException(
                     f"Get job execution error {job_id}: JobException: {err}"
                 ) from None
-            else:
-                raise NotImplementedError() from None
+            raise NotImplementedError(
+                "Handle error from the job execution does not support yet."
+            ) from None
 
         return Result(status=0, context=params).set_run_id(run_id)
 
     def execute(
         self,
         params: DictData,
-        run_id: str | None = None,
         *,
-        timeout: int = 60,
+        run_id: str | None = None,
+        timeout: int = 0,
     ) -> Result:
         """Execute workflow with passing a dynamic parameters to all jobs that
         included in this workflow model with ``jobs`` field.
@@ -712,13 +750,13 @@ class Workflow(BaseModel):
 
         :param params: An input parameters that use on workflow execution that
             will parameterize before using it. Default is None.
-        :type params: DictData | None
+        :type params: DictData
+
         :param run_id: A workflow running ID for this job execution.
-        :type run_id: str | None
+        :type run_id: str | None (default: None)
         :param timeout: A workflow execution time out in second unit that use
-            for limit time of execution and waiting job dependency. Default is
-            60 seconds.
-        :type timeout: int
+            for limit time of execution and waiting job dependency.
+        :type timeout: int (default: 0)
 
         :rtype: Result
         """
@@ -769,17 +807,16 @@ class Workflow(BaseModel):
                     context=context,
                     ts=ts,
                     job_queue=jq,
-                    worker=config.max_job_parallel,
                     timeout=timeout,
                 )
         except WorkflowException as err:
+            status: int = 1
             context.update(
                 {
                     "error": err,
                     "error_message": f"{err.__class__.__name__}: {err}",
                 },
             )
-            status = 1
         return rs.catch(status=status, context=context)
 
     def __exec_threading(
@@ -789,10 +826,10 @@ class Workflow(BaseModel):
         ts: float,
         job_queue: Queue,
         *,
-        worker: int = 2,
-        timeout: int = 600,
+        timeout: int = 0,
+        thread_timeout: int = 1800,
     ) -> DictData:
-        """Workflow execution by threading strategy.
+        """Workflow execution by threading strategy that use multithreading.
 
             If a job need dependency, it will check dependency job ID from
         context data before allow it run.
@@ -802,22 +839,26 @@ class Workflow(BaseModel):
             timeout.
         :param job_queue: A job queue object.
         :param timeout: A second value unit that bounding running time.
-        :param worker: A number of threading executor pool size.
+        :param thread_timeout: A timeout to waiting all futures complete.
+
         :rtype: DictData
         """
-        not_time_out_flag: bool = True
+        not_timeout_flag: bool = True
+        timeout: int = timeout or config.max_job_exec_timeout
         logger.debug(
-            f"({run_id}): [WORKFLOW]: Run {self.name} with threading job "
-            f"executor"
+            f"({run_id}) [WORKFLOW]: Run {self.name} with threading executor."
         )
 
         # IMPORTANT: The job execution can run parallel and waiting by
         #   needed.
-        with ThreadPoolExecutor(max_workers=worker) as executor:
+        with ThreadPoolExecutor(
+            max_workers=config.max_job_parallel,
+            thread_name_prefix="workflow_exec_threading_",
+        ) as executor:
             futures: list[Future] = []
 
             while not job_queue.empty() and (
-                not_time_out_flag := ((time.monotonic() - ts) < timeout)
+                not_timeout_flag := ((time.monotonic() - ts) < timeout)
             ):
                 job_id: str = job_queue.get()
                 job: Job = self.jobs[job_id]
@@ -846,29 +887,31 @@ class Workflow(BaseModel):
                 # NOTE: Mark this job queue done.
                 job_queue.task_done()
 
-            # NOTE: Wait for all items to finish processing
-            job_queue.join()
+            if not_timeout_flag:
 
-            for future in as_completed(futures, timeout=1800):
-                if err := future.exception():
-                    logger.error(f"({run_id}) [WORKFLOW]: {err}")
-                    raise WorkflowException(f"{err}")
-                try:
-                    future.result(timeout=60)
-                except TimeoutError as err:  # pragma: no cove
-                    raise WorkflowException(
-                        "Timeout when getting result from future"
-                    ) from err
-
-        if not_time_out_flag:
-            return context
+                # NOTE: Wait for all items to finish processing by `task_done()`
+                #   method.
+                job_queue.join()
+
+                for future in as_completed(futures, timeout=thread_timeout):
+                    if err := future.exception():
+                        logger.error(f"({run_id}) [WORKFLOW]: {err}")
+                        raise WorkflowException(f"{err}")
+
+                    # NOTE: This getting result does not do anything.
+                    future.result()
+
+                return context
+
+            for future in futures:
+                future.cancel()
 
         # NOTE: Raise timeout error.
-        logger.warning(  # pragma: no cov
-            f"({run_id}) [WORKFLOW]: Execution of workflow, {self.name!r} "
-            f", was timeout"
+        logger.warning(
+            f"({run_id}) [WORKFLOW]: Execution of workflow, {self.name!r}, "
+            f"was timeout"
         )
-        raise WorkflowException(  # pragma: no cov
+        raise WorkflowException(
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -879,7 +922,7 @@ class Workflow(BaseModel):
         ts: float,
         job_queue: Queue,
         *,
-        timeout: int = 600,
+        timeout: int = 0,
     ) -> DictData:
         """Workflow execution with non-threading strategy that use sequential
         job running and waiting previous job was run successful.
@@ -891,16 +934,18 @@ class Workflow(BaseModel):
         :param ts: A start timestamp that use for checking execute time should
             timeout.
         :param timeout: A second value unit that bounding running time.
+
         :rtype: DictData
         """
-        not_time_out_flag: bool = True
+        not_timeout_flag: bool = True
+        timeout: int = timeout or config.max_job_exec_timeout
         logger.debug(
-            f"({run_id}) [WORKFLOW]: Run {self.name} with non-threading job "
-            f"executor"
+            f"({run_id}) [WORKFLOW]: Run {self.name} with non-threading "
+            f"executor."
         )
 
         while not job_queue.empty() and (
-            not_time_out_flag := ((time.monotonic() - ts) < timeout)
+            not_timeout_flag := ((time.monotonic() - ts) < timeout)
         ):
             job_id: str = job_queue.get()
             job: Job = self.jobs[job_id]
@@ -909,7 +954,7 @@ class Workflow(BaseModel):
             if any(need not in context["jobs"] for need in job.needs):
                 job_queue.task_done()
                 job_queue.put(job_id)
-                time.sleep(0.05)
+                time.sleep(0.075)
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
@@ -925,27 +970,29 @@ class Workflow(BaseModel):
             # NOTE: Mark this job queue done.
             job_queue.task_done()
 
-        # NOTE: Wait for all items to finish processing
-        job_queue.join()
+        if not_timeout_flag:
+
+            # NOTE: Wait for all items to finish processing by `task_done()`
+            #   method.
+            job_queue.join()
 
-        if not_time_out_flag:
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(  # pragma: no cov
+        logger.warning(
             f"({run_id}) [WORKFLOW]: Execution of workflow was timeout"
         )
-        raise WorkflowException(  # pragma: no cov
+        raise WorkflowException(
             f"Execution of workflow: {self.name} was timeout"
         )
 
 
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class WorkflowTaskData:
-    """Workflow task dataclass that use to keep mapping data and objects for
-    passing in multithreading task.
+    """Workflow task Pydantic dataclass object that use to keep mapping data and
+    workflow model for passing to the multithreading task.
 
-        This dataclass will be 1-1 mapping with workflow and cron runner
+        This dataclass object is mapping 1-to-1 with workflow and cron runner
     objects.
     """
 
@@ -1075,7 +1122,8 @@ class WorkflowTaskData:
         # NOTE: Queue next release date.
         logger.debug(f"[CORE]: {'-' * 100}")
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other: WorkflowTaskData) -> bool:
+        """Override equal property that will compare only the same type."""
         if isinstance(other, WorkflowTaskData):
             return (
                 self.workflow.name == other.workflow.name

{ddeutil_workflow-0.0.20.dist-info → ddeutil_workflow-0.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.20
+Version: 0.0.21
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -195,6 +195,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_JOB_RAISE_ERROR`         | Core      | true                             | A flag that all job raise JobException from job strategy execution.                                                |        |
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
+| `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

{ddeutil_workflow-0.0.20.dist-info → ddeutil_workflow-0.0.21.dist-info}/RECORD

@@ -1,22 +1,22 @@
-ddeutil/workflow/__about__.py,sha256=Ktc8H9ZIafIIgqUKw2AXtyKPjKSj3ZNsWqgH0Gs_vow,28
+ddeutil/workflow/__about__.py,sha256=at-1IOQn4CID6hNRKtCzRBtCjVNyizp3IXd27XWqHPQ,28
 ddeutil/workflow/__cron.py,sha256=_2P9nmGOwGdv5bLgf9TpML2HBgqLv_qRgiO1Rulo1PA,26693
 ddeutil/workflow/__init__.py,sha256=DCSN0foPFlFLN_Q4uoWa_EBBlKeMHXGpOdr-lWHISrQ,1422
-ddeutil/workflow/__types.py,sha256=yizLXzjQpBt_WPaof2pIyncitJvYeksw4Q1zYJeuCLA,3707
+ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
 ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
 ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
-ddeutil/workflow/conf.py,sha256=ojFXYn1n8kBxl6wzEeVcHVmY8eFfnAd1AMt1zkNASmA,15617
+ddeutil/workflow/conf.py,sha256=KSwEHlZ_2I-bWGNKllDvkxNLy1WdHjUGxGH45_A3K5w,15717
 ddeutil/workflow/exceptions.py,sha256=NqnQJP52S59XIYMeXbTDbr4xH2UZ5EA3ejpU5Z4g6cQ,894
-ddeutil/workflow/job.py,sha256=wv_ybLKNCzMIoNJwyFV2aUNAA7X3UfidXo1E7pESduQ,24455
+ddeutil/workflow/job.py,sha256=jj8f8SyFD6GQyTZnxhviiDGv2ELVybMmVDDoxORoy1A,23875
 ddeutil/workflow/on.py,sha256=3Typ9YS2303LTijGK4ytN38ZLC0Gyq55HtFd0lm97Ic,7391
 ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
 ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
 ddeutil/workflow/scheduler.py,sha256=B2uXsqzmp32nIbya8EDePYyRhpwcxCMeoibPABCuMOA,18750
-ddeutil/workflow/stage.py,sha256=VRiKnq73wzC5GZOHNzcjrJXCyyCHmnSXvCFh3nySpSk,26339
+ddeutil/workflow/stage.py,sha256=Tzq-ciMZXKNUZ4cH4djyPIZ8aqj_P1Qm5zYZqYF8bDM,26301
 ddeutil/workflow/utils.py,sha256=IUTj7c6Jsi1oNHP7inLpv1TYhAA44lEMU1n5nNa1-bk,25657
-ddeutil/workflow/workflow.py,sha256=qztEyaUmSk0jXBfWhU0h9QHMk2W5rPRgEXZTjJN0nXA,37037
-ddeutil_workflow-0.0.20.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.20.dist-info/METADATA,sha256=5pjsJTd32g0Cm3VawZYRJAET-0xI2iP-gWrYgfDTZxU,13583
-ddeutil_workflow-0.0.20.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ddeutil_workflow-0.0.20.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
-ddeutil_workflow-0.0.20.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.20.dist-info/RECORD,,
+ddeutil/workflow/workflow.py,sha256=w800yMcajcIxdhWXgmqtPYGiiU2ftwsjyqqqnh-1-7o,38405
+ddeutil_workflow-0.0.21.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.21.dist-info/METADATA,sha256=7JQpnRPdK6pOi2tzqK8_dquR5LHSlD0OP8agmVxCEUg,13800
+ddeutil_workflow-0.0.21.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ddeutil_workflow-0.0.21.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.21.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.21.dist-info/RECORD,,