ddeutil-workflow 0.0.21__py3-none-any.whl → 0.0.22__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.21"
+__version__: str = "0.0.22"

ddeutil/workflow/conf.py

@@ -106,6 +106,9 @@ class Config:
     max_poking_pool_worker: int = int(
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )
+    max_on_per_workflow: int = int(
+        env("WORKFLOW_CORE_MAX_ON_PER_WORKFLOW", "5")
+    )
 
     # NOTE: Schedule App
     max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
@@ -462,6 +465,7 @@ class FileLog(BaseLog):
 
         :param excluded: An excluded list of key name that want to pass in the
             model_dump method.
+
         :rtype: Self
         """
         # NOTE: Check environ variable was set for real writing.

ddeutil/workflow/job.py

@@ -22,7 +22,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
@@ -40,6 +40,7 @@ from .stage import Stage
 from .utils import (
     Result,
     cross_product,
+    cut_id,
     dash2underscore,
     filter_func,
     gen_id,
@@ -346,6 +347,13 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
+    def check_needs(self, jobs: dict[str, Any]) -> bool:
+        """Return True if job's need exists in an input list of job's ID.
+
+        :rtype: bool
+        """
+        return all(need in jobs for need in self.needs)
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the receive context. The
         result from execution will pass to value of ``strategies`` key.
@@ -427,6 +435,7 @@ class Job(BaseModel):
         """
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
+        rs: Result = Result(run_id=run_id)
 
         # PARAGRAPH:
         #
@@ -447,14 +456,18 @@ class Job(BaseModel):
         for stage in self.stages:
 
             if stage.is_skipped(params=context):
-                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
+                logger.info(
+                    f"({cut_id(run_id)}) [JOB]: Skip stage: {stage.iden!r}"
+                )
                 continue
 
-            logger.info(f"({run_id}) [JOB]: Execute stage: {stage.iden!r}")
+            logger.info(
+                f"({cut_id(run_id)}) [JOB]: Execute stage: {stage.iden!r}"
+            )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({run_id}) [JOB]: ... Matrix: {strategy}")
+                logger.info(f"({cut_id(run_id)}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -463,7 +476,7 @@ class Job(BaseModel):
                     "Job strategy was canceled from event that had set before "
                     "strategy execution."
                 )
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -478,7 +491,6 @@ class Job(BaseModel):
                             "error_message": error_msg,
                         },
                     },
-                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -506,14 +518,14 @@ class Job(BaseModel):
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -523,13 +535,12 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
-                    run_id=run_id,
                 )
 
             # NOTE: Remove the current stage object for saving memory.
             del stage
 
-        return Result(
+        return rs.catch(
             status=0,
             context={
                 strategy_id: {
@@ -537,7 +548,6 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
-            run_id=run_id,
         )
 
     def execute(self, params: DictData, run_id: str | None = None) -> Result:
@@ -619,7 +629,7 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
@@ -631,7 +641,7 @@ class Job(BaseModel):
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({cut_id(run_id)}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -649,7 +659,7 @@ class Job(BaseModel):
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({run_id}) [JOB]: Fail-fast catching:\n\t"
+                    f"({cut_id(run_id)}) [JOB]: Fail-fast catching:\n\t"
                     f"{future.exception()}"
                 )
                 context.update(
@@ -680,7 +690,7 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
@@ -690,7 +700,7 @@ class Job(BaseModel):
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({run_id}) [JOB]: All-completed catching:\n\t"
+                    f"({cut_id(run_id)}) [JOB]: All-completed catching:\n\t"
                     f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(

ddeutil/workflow/on.py

@@ -189,7 +189,10 @@ class On(BaseModel):
         date that given from input.
         """
         runner: CronRunner = self.generate(start=start)
+
+        # NOTE: ship the next date of runner object that create from start.
         _ = runner.next
+
         return runner
 
 

ddeutil/workflow/stage.py

@@ -55,6 +55,7 @@ from .utils import (
     Registry,
     Result,
     TagFunc,
+    cut_id,
     gen_id,
     make_exec,
     make_registry,
@@ -124,13 +125,16 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 run_id: str = gen_id(self.name + (self.id or ""), unique=True)
                 kwargs["run_id"] = run_id
 
+            rs_raise: Result = Result(status=1, run_id=run_id)
+
             try:
                 # NOTE: Start calling origin function with a passing args.
                 return func(self, *args, **kwargs)
             except Exception as err:
                 # NOTE: Start catching error from the stage execution.
                 logger.error(
-                    f"({run_id}) [STAGE]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
+                    f"{err}"
                 )
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
@@ -147,13 +151,12 @@ def handler_result(message: str | None = None) -> DecoratorResult:
 
                 # NOTE: Catching exception error object to result with
                 #   error_message and error keys.
-                return Result(
+                return rs_raise.catch(
                     status=1,
                     context={
                         "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                    run_id=run_id,
                 )
 
         return wrapped
@@ -339,7 +342,7 @@ class EmptyStage(BaseStage):
         :rtype: Result
         """
         logger.info(
-            f"({run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
+            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
@@ -393,7 +396,9 @@ class BashStage(BaseStage):
         f_name: str = f"{run_id}.sh"
         f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
 
-        logger.debug(f"({run_id}) [STAGE]: Start create `{f_name}` file.")
+        logger.debug(
+            f"({cut_id(run_id)}) [STAGE]: Start create `{f_name}` file."
+        )
 
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: write header of `.sh` file
@@ -425,7 +430,7 @@ class BashStage(BaseStage):
         """
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({run_id}) [STAGE]: Shell-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
             bash=bash, env=param2template(self.env, params), run_id=run_id
         ) as sh:
@@ -535,7 +540,7 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({run_id}) [STAGE]: Py-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is vary dangerous. So, it
         #   should us the re module to validate exec-string before running.
@@ -660,7 +665,8 @@ class HookStage(BaseStage):
                 args[k] = args.pop(k.removeprefix("_"))
 
         logger.info(
-            f"({run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
+            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
+            f"{t_func.name}@{t_func.tag}"
         )
         rs: DictData = t_func(**param2template(args, params))
 
@@ -716,7 +722,9 @@ class TriggerStage(BaseStage):
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
         wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(f"({run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
+        logger.info(
+            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
+        )
         return wf.execute(
             params=param2template(self.params, params),
             run_id=run_id,

ddeutil/workflow/utils.py

@@ -430,7 +430,7 @@ class Result:
         return self
 
 
-def make_exec(path: str | Path) -> None:  # pragma: no cov
+def make_exec(path: str | Path) -> None:
     """Change mode of file to be executable file.
 
     :param path: A file path that want to make executable permission.
@@ -451,7 +451,9 @@ FILTERS: dict[str, callable] = {  # pragma: no cov
 
 
 class FilterFunc(Protocol):
-    """Tag Function Protocol"""
+    """Tag Function Protocol. This protocol that use to represent any callable
+    object that able to access the name attribute.
+    """
 
     name: str
 
@@ -814,3 +816,17 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
 
 def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
     return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
+
+
+def cut_id(run_id: str, *, num: int = 6):
+    """Cutting running ID with length.
+
+    Example:
+        >>> cut_id(run_id='668931127320241228100331254567')
+        '254567'
+
+    :param run_id:
+    :param num:
+    :return:
+    """
+    return run_id[-num:]

ddeutil/workflow/workflow.py

@@ -49,6 +49,7 @@ from .on import On
 from .utils import (
     Param,
     Result,
+    cut_id,
     delay,
     gen_id,
     get_diff_sec,
@@ -123,12 +124,38 @@ class WorkflowRelease:
 
 @dataclass
 class WorkflowQueue:
-    """Workflow Queue object."""
+    """Workflow Queue object that is management of WorkflowRelease objects."""
 
     queue: list[WorkflowRelease] = field(default_factory=list)
     running: list[WorkflowRelease] = field(default_factory=list)
     complete: list[WorkflowRelease] = field(default_factory=list)
 
+    @classmethod
+    def from_list(
+        cls, queue: list[datetime] | list[WorkflowRelease] | None
+    ) -> Self:
+        """Construct WorkflowQueue object from an input queue value that passing
+        with list of datetime or list of WorkflowRelease.
+
+        :raise TypeError: If the type of an input queue does not valid.
+
+        :rtype: Self
+        """
+        if queue is None:
+            return cls()
+        elif isinstance(queue, list):
+
+            if all(isinstance(q, datetime) for q in queue):
+                return cls(queue=[WorkflowRelease.from_dt(q) for q in queue])
+
+            elif all(isinstance(q, WorkflowRelease) for q in queue):
+                return cls(queue=queue)
+
+        raise TypeError(
+            "Type of the queue does not valid with WorkflowQueue "
+            "or list of datetime or list of WorkflowRelease."
+        )
+
     @property
     def is_queued(self) -> bool:
         """Return True if it has workflow release object in the queue.
@@ -137,34 +164,35 @@ class WorkflowQueue:
         """
         return len(self.queue) > 0
 
-    def check_queue(self, data: WorkflowRelease) -> bool:
+    def check_queue(self, value: WorkflowRelease) -> bool:
         """Check a WorkflowRelease value already exists in list of tracking
         queues.
 
-        :param data: A workflow release object.
+        :param value: A WorkflowRelease object that want to check it already in
+            queues.
 
         :rtype: bool
         """
         return (
-            (data in self.queue)
-            or (data in self.running)
-            or (data in self.complete)
+            (value in self.queue)
+            or (value in self.running)
+            or (value in self.complete)
         )
 
-    def push_queue(self, data: WorkflowRelease) -> Self:
+    def push_queue(self, value: WorkflowRelease) -> Self:
         """Push data to the queue."""
-        heappush(self.queue, data)
+        heappush(self.queue, value)
         return self
 
-    def push_running(self, data: WorkflowRelease) -> Self:
+    def push_running(self, value: WorkflowRelease) -> Self:
         """Push data to the running."""
-        heappush(self.running, data)
+        heappush(self.running, value)
         return self
 
-    def remove_running(self, data: WorkflowRelease) -> Self:
+    def remove_running(self, value: WorkflowRelease) -> Self:
         """Remove data on the running if it exists."""
-        if data in self.running:
-            self.running.remove(data)
+        if value in self.running:
+            self.running.remove(value)
 
 
 class Workflow(BaseModel):
@@ -283,7 +311,7 @@ class Workflow(BaseModel):
         return dedent(value)
 
     @field_validator("on", mode="after")
-    def __on_no_dup__(cls, value: list[On]) -> list[On]:
+    def __on_no_dup_and_reach_limit__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
         contain the every minute value more than one value, it will remove to
         only one value.
@@ -306,6 +334,12 @@ class Workflow(BaseModel):
         #         "If it has every minute cronjob on value, it should has only "
         #         "one value in the on field."
         #     )
+
+        if len(set_ons) > config.max_on_per_workflow:
+            raise ValueError(
+                f"The number of the on should not more than "
+                f"{config.max_on_per_workflow} crontab."
+            )
         return value
 
     @model_validator(mode="after")
@@ -404,22 +438,21 @@ class Workflow(BaseModel):
         self,
         release: datetime | WorkflowRelease,
         params: DictData,
-        run_id: str | None = None,
         *,
+        run_id: str | None = None,
         log: type[Log] = None,
-        queue: WorkflowQueue | list[datetime] | None = None,
+        queue: (
+            WorkflowQueue | list[datetime] | list[WorkflowRelease] | None
+        ) = None,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
         release templating that include logical date (release date), execution
         date, or running id to the params.
 
             This method allow workflow use log object to save the execution
-        result to log destination like file log to local `/logs` directory.
-
-            I will add sleep with 0.15 seconds on every step that interact with
-        the queue object.
+        result to log destination like file log to the local `/logs` directory.
 
-        :param release: A release datetime.
+        :param release: A release datetime or WorkflowRelease object.
         :param params: A workflow parameter that pass to execute method.
         :param queue: A list of release time that already queue.
         :param run_id: A workflow running ID for this release.
@@ -430,23 +463,23 @@ class Workflow(BaseModel):
         """
         log: type[Log] = log or FileLog
         run_id: str = run_id or gen_id(self.name, unique=True)
+        rs_release: Result = Result(run_id=run_id)
 
         # VALIDATE: Change queue value to WorkflowQueue object.
-        if queue is None:
-            queue: WorkflowQueue = WorkflowQueue()
-        elif isinstance(queue, list):
-            queue: WorkflowQueue = WorkflowQueue(queue=queue)
+        if queue is None or isinstance(queue, list):
+            queue: WorkflowQueue = WorkflowQueue.from_list(queue)
 
         # VALIDATE: Change release value to WorkflowRelease object.
         if isinstance(release, datetime):
             release: WorkflowRelease = WorkflowRelease.from_dt(release)
 
         logger.debug(
-            f"({run_id}) [RELEASE]: {self.name!r} : "
-            f"Closely to run >> {release.date:%Y-%m-%d %H:%M:%S}"
+            f"({cut_id(run_id)}) [RELEASE]: {self.name!r} : Start release - "
+            f"{release.date:%Y-%m-%d %H:%M:%S}"
         )
 
-        # NOTE: Release parameter that use to change if params has templating.
+        # NOTE: Release parameters that use to templating on the schedule
+        #   config data.
         release_params: DictData = {
             "release": {
                 "logical_date": release.date,
@@ -456,14 +489,14 @@ class Workflow(BaseModel):
             }
         }
 
-        # WARNING: Re-create workflow object that use new running workflow ID.
+        # NOTE: Execute workflow with templating params from release mapping.
         rs: Result = self.execute(
             params=param2template(params, release_params),
             run_id=run_id,
         )
         logger.debug(
-            f"({run_id}) [RELEASE]: {self.name!r} : "
-            f"End release {release.date:%Y-%m-%d %H:%M:%S}"
+            f"({cut_id(run_id)}) [RELEASE]: {self.name!r} : End release - "
+            f"{release.date:%Y-%m-%d %H:%M:%S}"
         )
 
         rs.set_parent_run_id(run_id)
@@ -485,16 +518,13 @@ class Workflow(BaseModel):
         queue.remove_running(release)
         heappush(queue.complete, release)
 
-        return Result(
+        return rs_release.catch(
             status=0,
             context={
                 "params": params,
-                "release": {
-                    "status": "success",
-                    "logical_date": release.date,
-                },
+                "release": {"status": "success", "logical_date": release.date},
+                "outputs": rs.context,
             },
-            run_id=run_id,
         )
 
     def queue_poking(
@@ -503,14 +533,20 @@ class Workflow(BaseModel):
         end_date: datetime,
         queue: WorkflowQueue,
         log: type[Log],
+        *,
+        force_run: bool = False,
     ) -> WorkflowQueue:
         """Generate queue of datetime from the cron runner that initialize from
         the on field. with offset value.
 
-        :param offset:
-        :param end_date:
-        :param queue:
-        :param log:
+        :param offset: A offset in second unit for time travel.
+        :param end_date: An end datetime object.
+        :param queue: A workflow queue object.
+        :param log: A log class that want to making log object.
+        :param force_run: A flag that allow to release workflow if the log with
+            that release was pointed.
+
+        :rtype: WorkflowQueue
         """
         for on in self.on:
 
@@ -518,6 +554,7 @@ class Workflow(BaseModel):
                 get_dt_now(tz=config.tz, offset=offset).replace(microsecond=0)
             )
 
+            # NOTE: Skip this runner date if it more than the end date.
             if runner.date > end_date:
                 continue
 
@@ -529,8 +566,9 @@ class Workflow(BaseModel):
                 type="poking",
             )
 
-            while queue.check_queue(data=workflow_release) or (
+            while queue.check_queue(workflow_release) or (
                 log.is_pointed(name=self.name, release=workflow_release.date)
+                and not force_run
             ):
                 workflow_release = WorkflowRelease(
                     date=runner.next,
@@ -543,36 +581,50 @@ class Workflow(BaseModel):
             if runner.date > end_date:
                 continue
 
+            # NOTE: Push the WorkflowRelease object to queue.
             queue.push_queue(workflow_release)
+
         return queue
 
     def poke(
         self,
         start_date: datetime | None = None,
         params: DictData | None = None,
+        *,
         run_id: str | None = None,
         periods: int = 1,
-        *,
         log: Log | None = None,
+        force_run: bool = False,
+        timeout: int = 1800,
     ) -> list[Result]:
-        """Poke workflow with the ``on`` field with threading executor pool for
-        executing with all its schedules that was set on the `on` value.
-        This method will observe its schedule that nearing to run with the
+        """Poke this workflow with start datetime value that passing to its
+        ``on`` field with threading executor pool for executing with all its
+        schedules that was set on the `on` value.
+
+            This method will observe its schedule that nearing to run with the
         ``self.release()`` method.
 
         :param start_date: A start datetime object.
         :param params: A parameters that want to pass to the release method.
         :param run_id: A workflow running ID for this poke.
-        :param periods: A periods of minutes value to running poke.
+        :param periods: A periods in minutes value that use to run this poking.
         :param log: A log object that want to use on this poking process.
+        :param force_run: A flag that allow to release workflow if the log with
+            that release was pointed.
+        :param timeout: A second value for timeout while waiting all futures
+            run completely.
 
         :rtype: list[Result]
+        :return: A list of all results that return from ``self.release`` method.
         """
+        log: type[Log] = log or FileLog
+        run_id: str = run_id or gen_id(self.name, unique=True)
+
         # NOTE: If this workflow does not set the on schedule, it will return
         #   empty result.
         if len(self.on) == 0:
             logger.info(
-                f"({run_id}) [POKING]: {self.name!r} does not have any "
+                f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have any "
                 f"schedule to run."
             )
             return []
@@ -592,82 +644,94 @@ class Workflow(BaseModel):
             start_date: datetime = current_date
             offset: float = 0
 
+        # NOTE: End date is use to stop generate queue with an input periods
+        #   value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
-        log: type[Log] = log or FileLog
-        run_id: str = run_id or gen_id(self.name, unique=True)
         logger.info(
-            f"({run_id}) [POKING]: Start Poking: {self.name!r} from "
+            f"({cut_id(run_id)}) [POKING]: Start Poking: {self.name!r} from "
             f"{start_date:%Y-%m-%d %H:%M:%S} to {end_date:%Y-%m-%d %H:%M:%S}"
         )
 
-        params: DictData = params or {}
-        workflow_queue: WorkflowQueue = WorkflowQueue()
+        params: DictData = {} if params is None else params
+        wf_queue: WorkflowQueue = WorkflowQueue()
         results: list[Result] = []
         futures: list[Future] = []
 
+        # NOTE: Make queue to the workflow queue object.
         self.queue_poking(
-            offset, end_date=end_date, queue=workflow_queue, log=log
+            offset,
+            end_date=end_date,
+            queue=wf_queue,
+            log=log,
+            force_run=force_run,
         )
-
-        if len(workflow_queue.queue) == 0:
+        if not wf_queue.is_queued:
             logger.info(
-                f"({run_id}) [POKING]: {self.name!r} does not have any "
-                f"queue to run."
+                f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have "
+                f"any queue."
             )
             return []
 
+        # NOTE: Start create the thread pool executor for running this poke
+        #   process.
         with ThreadPoolExecutor(
             max_workers=config.max_poking_pool_worker,
-            thread_name_prefix="workflow_poking_",
+            thread_name_prefix="wf_poking_",
         ) as executor:
 
-            while workflow_queue.is_queued:
+            while wf_queue.is_queued:
+
+                # NOTE: Pop the latest WorkflowRelease object from queue.
+                release: WorkflowRelease = heappop(wf_queue.queue)
 
-                wf_release: WorkflowRelease = heappop(workflow_queue.queue)
                 if (
-                    wf_release.date - get_dt_now(tz=config.tz, offset=offset)
+                    release.date - get_dt_now(tz=config.tz, offset=offset)
                 ).total_seconds() > 60:
                     logger.debug(
-                        f"({run_id}) [POKING]: Waiting because the latest "
-                        f"release has diff time more than 60 seconds "
+                        f"({cut_id(run_id)}) [POKING]: Wait because the latest "
+                        f"release has diff time more than 60 seconds ..."
                     )
-                    heappush(workflow_queue.queue, wf_release)
+                    heappush(wf_queue.queue, release)
                     delay(60)
+
+                    # WARNING: I already call queue poking again because issue
+                    #   about the every minute crontab.
                     self.queue_poking(
-                        offset, end_date, queue=workflow_queue, log=log
+                        offset,
+                        end_date,
+                        queue=wf_queue,
+                        log=log,
+                        force_run=force_run,
                     )
                     continue
 
-                # NOTE: Push the workflow release to running queue
-                workflow_queue.push_running(wf_release)
+                # NOTE: Push the latest WorkflowRelease to the running queue.
+                wf_queue.push_running(release)
 
                 futures.append(
                     executor.submit(
                         self.release,
-                        release=wf_release,
+                        release=release,
                         params=params,
                         log=log,
-                        queue=workflow_queue,
+                        queue=wf_queue,
                     )
                 )
 
                 self.queue_poking(
-                    offset, end_date, queue=workflow_queue, log=log
+                    offset,
+                    end_date,
+                    queue=wf_queue,
+                    log=log,
+                    force_run=force_run,
                 )
 
             # WARNING: This poking method does not allow to use fail-fast
             #   logic to catching parallel execution result.
-            for future in as_completed(futures):
+            for future in as_completed(futures, timeout=timeout):
                 results.append(future.result().set_parent_run_id(run_id))
 
-        while len(workflow_queue.running) > 0:  # pragma: no cov
-            logger.warning(
-                f"({run_id}) [POKING]: Running does empty when poking "
-                f"process was finishing."
-            )
-            delay(10)
-
         return results
 
     def execute_job(
@@ -685,6 +749,8 @@ class Workflow(BaseModel):
         model. It different with ``self.execute`` because this method run only
         one job and return with context of this job data.
 
+        :raise WorkflowException: If execute with not exist job's ID.
+        :raise WorkflowException: If the job execution raise JobException.
         :raise NotImplementedError: If set raise_error argument to False.
 
         :param job_id: A job ID that want to execute.
@@ -694,8 +760,11 @@ class Workflow(BaseModel):
             if it get exception from job execution.
 
         :rtype: Result
+        :return: Return the result object that receive the job execution result
+            context.
         """
         run_id: str = run_id or gen_id(self.name, unique=True)
+        rs: Result = Result(run_id=run_id)
 
         # VALIDATE: check a job ID that exists in this workflow or not.
         if job_id not in self.jobs:
@@ -704,10 +773,14 @@ class Workflow(BaseModel):
                 f"workflow."
             )
 
-        logger.info(f"({run_id}) [WORKFLOW]: Start execute job: {job_id!r}")
+        logger.info(
+            f"({cut_id(run_id)}) [WORKFLOW]: Start execute job: {job_id!r}"
+        )
 
         # IMPORTANT:
-        #   Change any job running IDs to this workflow running ID.
+        #   This execution change all job running IDs to the current workflow
+        #   execution running ID (with passing run_id to the job execution
+        #   argument).
         #
         try:
             job: Job = self.jobs[job_id]
@@ -717,7 +790,8 @@ class Workflow(BaseModel):
             )
         except JobException as err:
             logger.error(
-                f"({run_id}) [WORKFLOW]: {err.__class__.__name__}: {err}"
+                f"({cut_id(run_id)}) [WORKFLOW]: {err.__class__.__name__}: "
+                f"{err}"
             )
             if raise_error:
                 raise WorkflowException(
@@ -727,7 +801,7 @@ class Workflow(BaseModel):
                 "Handle error from the job execution does not support yet."
             ) from None
 
-        return Result(status=0, context=params).set_run_id(run_id)
+        return rs.catch(status=0, context=params)
 
     def execute(
         self,
@@ -761,7 +835,9 @@ class Workflow(BaseModel):
         :rtype: Result
         """
         run_id: str = run_id or gen_id(self.name, unique=True)
-        logger.info(f"({run_id}) [WORKFLOW]: Start Execute: {self.name!r} ...")
+        logger.info(
+            f"({cut_id(run_id)}) [WORKFLOW]: Start Execute: {self.name!r} ..."
+        )
 
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
@@ -771,7 +847,7 @@ class Workflow(BaseModel):
         # NOTE: It should not do anything if it does not have job.
         if not self.jobs:
             logger.warning(
-                f"({run_id}) [WORKFLOW]: This workflow: {self.name!r} "
+                f"({cut_id(run_id)}) [WORKFLOW]: This workflow: {self.name!r} "
                 f"does not have any jobs"
             )
             return rs.catch(status=0, context=params)
@@ -846,14 +922,14 @@ class Workflow(BaseModel):
         not_timeout_flag: bool = True
         timeout: int = timeout or config.max_job_exec_timeout
         logger.debug(
-            f"({run_id}) [WORKFLOW]: Run {self.name} with threading executor."
+            f"({cut_id(run_id)}) [WORKFLOW]: Run {self.name!r} with threading."
         )
 
         # IMPORTANT: The job execution can run parallel and waiting by
         #   needed.
         with ThreadPoolExecutor(
             max_workers=config.max_job_parallel,
-            thread_name_prefix="workflow_exec_threading_",
+            thread_name_prefix="wf_exec_threading_",
         ) as executor:
             futures: list[Future] = []
 
@@ -863,7 +939,7 @@ class Workflow(BaseModel):
                 job_id: str = job_queue.get()
                 job: Job = self.jobs[job_id]
 
-                if any(need not in context["jobs"] for need in job.needs):
+                if not job.check_needs(context["jobs"]):
                     job_queue.task_done()
                     job_queue.put(job_id)
                     time.sleep(0.25)
@@ -872,10 +948,13 @@ class Workflow(BaseModel):
                 # NOTE: Start workflow job execution with deep copy context data
                 #   before release.
                 #
+                #   Context:
+                #   ---
                 #   {
                 #       'params': <input-params>,
-                #       'jobs': {},
+                #       'jobs': { <job's-id>: ... },
                 #   }
+                #
                 futures.append(
                     executor.submit(
                         self.execute_job,
@@ -895,8 +974,8 @@ class Workflow(BaseModel):
 
                 for future in as_completed(futures, timeout=thread_timeout):
                     if err := future.exception():
-                        logger.error(f"({run_id}) [WORKFLOW]: {err}")
-                        raise WorkflowException(f"{err}")
+                        logger.error(f"({cut_id(run_id)}) [WORKFLOW]: {err}")
+                        raise WorkflowException(str(err))
 
                     # NOTE: This getting result does not do anything.
                     future.result()
@@ -908,12 +987,10 @@ class Workflow(BaseModel):
 
         # NOTE: Raise timeout error.
         logger.warning(
-            f"({run_id}) [WORKFLOW]: Execution of workflow, {self.name!r}, "
-            f"was timeout"
-        )
-        raise WorkflowException(
-            f"Execution of workflow: {self.name} was timeout"
+            f"({cut_id(run_id)}) [WORKFLOW]: Execution: {self.name!r} "
+            f"was timeout."
         )
+        raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
     def __exec_non_threading(
         self,
@@ -940,8 +1017,8 @@ class Workflow(BaseModel):
         not_timeout_flag: bool = True
         timeout: int = timeout or config.max_job_exec_timeout
         logger.debug(
-            f"({run_id}) [WORKFLOW]: Run {self.name} with non-threading "
-            f"executor."
+            f"({cut_id(run_id)}) [WORKFLOW]: Run {self.name} with "
+            f"non-threading."
         )
 
         while not job_queue.empty() and (
@@ -951,7 +1028,7 @@ class Workflow(BaseModel):
             job: Job = self.jobs[job_id]
 
             # NOTE: Waiting dependency job run successful before release.
-            if any(need not in context["jobs"] for need in job.needs):
+            if not job.check_needs(context["jobs"]):
                 job_queue.task_done()
                 job_queue.put(job_id)
                 time.sleep(0.075)
@@ -980,11 +1057,10 @@ class Workflow(BaseModel):
 
         # NOTE: Raise timeout error.
         logger.warning(
-            f"({run_id}) [WORKFLOW]: Execution of workflow was timeout"
-        )
-        raise WorkflowException(
-            f"Execution of workflow: {self.name} was timeout"
+            f"({cut_id(run_id)}) [WORKFLOW]: Execution: {self.name!r} "
+            f"was timeout."
         )
+        raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
 
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
@@ -1035,15 +1111,15 @@ class WorkflowTaskData:
             next_time: datetime = runner.next
 
         logger.debug(
-            f"({run_id}) [CORE]: {self.workflow.name!r} : {runner.cron} : "
-            f"{next_time:%Y-%m-%d %H:%M:%S}"
+            f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
+            f"{runner.cron} : {next_time:%Y-%m-%d %H:%M:%S}"
         )
         heappush(queue[self.alias], next_time)
         start_sec: float = time.monotonic()
 
         if get_diff_sec(next_time, tz=runner.tz) > waiting_sec:
             logger.debug(
-                f"({run_id}) [WORKFLOW]: {self.workflow.name!r} : "
+                f"({cut_id(run_id)}) [WORKFLOW]: {self.workflow.name!r} : "
                 f"{runner.cron} "
                 f": Does not closely >> {next_time:%Y-%m-%d %H:%M:%S}"
             )
@@ -1056,8 +1132,8 @@ class WorkflowTaskData:
             return
 
         logger.debug(
-            f"({run_id}) [CORE]: {self.workflow.name!r} : {runner.cron} : "
-            f"Closely to run >> {next_time:%Y-%m-%d %H:%M:%S}"
+            f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
+            f"{runner.cron} : Closely to run >> {next_time:%Y-%m-%d %H:%M:%S}"
         )
 
         # NOTE: Release when the time is nearly to schedule time.
@@ -1065,8 +1141,8 @@ class WorkflowTaskData:
             sleep_interval + 5
         ):
             logger.debug(
-                f"({run_id}) [CORE]: {self.workflow.name!r} : {runner.cron} "
-                f": Sleep until: {duration}"
+                f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
+                f"{runner.cron} : Sleep until: {duration}"
             )
             time.sleep(15)
 
@@ -1085,8 +1161,8 @@ class WorkflowTaskData:
             params=param2template(self.params, release_params),
         )
         logger.debug(
-            f"({run_id}) [CORE]: {self.workflow.name!r} : {runner.cron} : "
-            f"End release - {next_time:%Y-%m-%d %H:%M:%S}"
+            f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
+            f"{runner.cron} : End release - {next_time:%Y-%m-%d %H:%M:%S}"
         )
 
         # NOTE: Set parent ID on this result.

{ddeutil_workflow-0.0.21.dist-info → ddeutil_workflow-0.0.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.21
+Version: 0.0.22
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -196,6 +196,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
 | `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_ON_PER_WORKFLOW`     | Core      | 5                                |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

ddeutil_workflow-0.0.22.dist-info/RECORD

@@ -0,0 +1,22 @@
+ddeutil/workflow/__about__.py,sha256=hJavfsPLTnuXMwKFo9HZgsq6b7tJpWgyfttwaxzMujE,28
+ddeutil/workflow/__cron.py,sha256=_2P9nmGOwGdv5bLgf9TpML2HBgqLv_qRgiO1Rulo1PA,26693
+ddeutil/workflow/__init__.py,sha256=DCSN0foPFlFLN_Q4uoWa_EBBlKeMHXGpOdr-lWHISrQ,1422
+ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
+ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
+ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
+ddeutil/workflow/conf.py,sha256=GsbuJDQfQoAGiR4keUEoB4lKfZxdkaiZ4N4FfIHc0xY,15814
+ddeutil/workflow/exceptions.py,sha256=NqnQJP52S59XIYMeXbTDbr4xH2UZ5EA3ejpU5Z4g6cQ,894
+ddeutil/workflow/job.py,sha256=liu8M_pUhAGHZ_Ez922jI94LCC3yioI-Tw5o71Zy88w,24216
+ddeutil/workflow/on.py,sha256=wxKfL2u-bBhPbDtZbhqE2lZoPVukHA1zq-qrg0ldic0,7469
+ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
+ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
+ddeutil/workflow/scheduler.py,sha256=B2uXsqzmp32nIbya8EDePYyRhpwcxCMeoibPABCuMOA,18750
+ddeutil/workflow/stage.py,sha256=ADFqExFmD8Y00A86TSS05HpabvsLV7_dbLrzD31TkK8,26490
+ddeutil/workflow/utils.py,sha256=0GaHpRL1HuyES1NS7r56DFgloOVftYVAvAdVgIbPA_k,26001
+ddeutil/workflow/workflow.py,sha256=fzhKJx9s-RF95FQ0tAvAQ1nsL8dsp_py2Ea5TGnjsOk,41542
+ddeutil_workflow-0.0.22.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.22.dist-info/METADATA,sha256=J_VrfU8ZBPAa7OrhMV_c4sLbQ0g3Nc0MQSdYUgmxF6I,14017
+ddeutil_workflow-0.0.22.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ddeutil_workflow-0.0.22.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.22.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.22.dist-info/RECORD,,

ddeutil_workflow-0.0.21.dist-info/RECORD

@@ -1,22 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=at-1IOQn4CID6hNRKtCzRBtCjVNyizp3IXd27XWqHPQ,28
-ddeutil/workflow/__cron.py,sha256=_2P9nmGOwGdv5bLgf9TpML2HBgqLv_qRgiO1Rulo1PA,26693
-ddeutil/workflow/__init__.py,sha256=DCSN0foPFlFLN_Q4uoWa_EBBlKeMHXGpOdr-lWHISrQ,1422
-ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
-ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
-ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
-ddeutil/workflow/conf.py,sha256=KSwEHlZ_2I-bWGNKllDvkxNLy1WdHjUGxGH45_A3K5w,15717
-ddeutil/workflow/exceptions.py,sha256=NqnQJP52S59XIYMeXbTDbr4xH2UZ5EA3ejpU5Z4g6cQ,894
-ddeutil/workflow/job.py,sha256=jj8f8SyFD6GQyTZnxhviiDGv2ELVybMmVDDoxORoy1A,23875
-ddeutil/workflow/on.py,sha256=3Typ9YS2303LTijGK4ytN38ZLC0Gyq55HtFd0lm97Ic,7391
-ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
-ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
-ddeutil/workflow/scheduler.py,sha256=B2uXsqzmp32nIbya8EDePYyRhpwcxCMeoibPABCuMOA,18750
-ddeutil/workflow/stage.py,sha256=Tzq-ciMZXKNUZ4cH4djyPIZ8aqj_P1Qm5zYZqYF8bDM,26301
-ddeutil/workflow/utils.py,sha256=IUTj7c6Jsi1oNHP7inLpv1TYhAA44lEMU1n5nNa1-bk,25657
-ddeutil/workflow/workflow.py,sha256=w800yMcajcIxdhWXgmqtPYGiiU2ftwsjyqqqnh-1-7o,38405
-ddeutil_workflow-0.0.21.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.21.dist-info/METADATA,sha256=7JQpnRPdK6pOi2tzqK8_dquR5LHSlD0OP8agmVxCEUg,13800
-ddeutil_workflow-0.0.21.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ddeutil_workflow-0.0.21.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
-ddeutil_workflow-0.0.21.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.21.dist-info/RECORD,,