ddeutil-workflow 0.0.20__py3-none-any.whl → 0.0.22__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.20"
+__version__: str = "0.0.22"

ddeutil/workflow/__types.py

@@ -101,6 +101,7 @@ class Re:
     #
     #   Examples:
     #       - tasks/function@dummy
+    #
     __re_task_fmt: str = r"""
         ^
             (?P<path>[^/@]+)

ddeutil/workflow/conf.py

@@ -100,9 +100,15 @@ class Config:
 
     # NOTE: Workflow
     max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
+    max_job_exec_timeout: int = int(
+        env("WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT", "600")
+    )
     max_poking_pool_worker: int = int(
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )
+    max_on_per_workflow: int = int(
+        env("WORKFLOW_CORE_MAX_ON_PER_WORKFLOW", "5")
+    )
 
     # NOTE: Schedule App
     max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
@@ -459,6 +465,7 @@ class FileLog(BaseLog):
 
         :param excluded: An excluded list of key name that want to pass in the
             model_dump method.
+
         :rtype: Self
         """
         # NOTE: Check environ variable was set for real writing.

ddeutil/workflow/job.py

@@ -11,7 +11,6 @@ job.
 from __future__ import annotations
 
 import copy
-import time
 from concurrent.futures import (
     FIRST_EXCEPTION,
     Future,
@@ -23,7 +22,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
@@ -41,6 +40,7 @@ from .stage import Stage
 from .utils import (
     Result,
     cross_product,
+    cut_id,
     dash2underscore,
     filter_func,
     gen_id,
@@ -48,13 +48,13 @@ from .utils import (
 )
 
 logger = get_logger("ddeutil.workflow")
-MatrixInclude = list[dict[str, Union[str, int]]]
-MatrixExclude = list[dict[str, Union[str, int]]]
+MatrixFilter = list[dict[str, Union[str, int]]]
 
 
 __all__: TupleStr = (
     "Strategy",
     "Job",
+    "TriggerRules",
     "make",
 )
 
@@ -63,8 +63,8 @@ __all__: TupleStr = (
 @lru_cache
 def make(
     matrix: Matrix,
-    include: MatrixInclude,
-    exclude: MatrixExclude,
+    include: MatrixFilter,
+    exclude: MatrixFilter,
 ) -> list[DictStr]:
     """Make a list of product of matrix values that already filter with
     exclude matrix and add specific matrix with include.
@@ -124,7 +124,7 @@ def make(
 
 
 class Strategy(BaseModel):
-    """Strategy Model that will combine a matrix together for running the
+    """Strategy model that will combine a matrix together for running the
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
@@ -166,11 +166,11 @@ class Strategy(BaseModel):
             "A matrix values that want to cross product to possible strategies."
         ),
     )
-    include: MatrixInclude = Field(
+    include: MatrixFilter = Field(
         default_factory=list,
         description="A list of additional matrix that want to adds-in.",
     )
-    exclude: MatrixExclude = Field(
+    exclude: MatrixFilter = Field(
         default_factory=list,
         description="A list of exclude matrix that want to filter-out.",
     )
@@ -204,7 +204,7 @@ class Strategy(BaseModel):
 
 
 class TriggerRules(str, Enum):
-    """Trigger Rules enum object."""
+    """Trigger rules enum object."""
 
     all_success: str = "all_success"
     all_failed: str = "all_failed"
@@ -215,8 +215,15 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
+class RunsOn(str, Enum):
+    """Runs-On enum object."""
+
+    local: str = "local"
+    docker: str = "docker"
+
+
 class Job(BaseModel):
-    """Job Pydantic model object (group of stages).
+    """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
     you pass matrix mapping and it able to generate, you will see it running
@@ -327,7 +334,10 @@ class Job(BaseModel):
         return self
 
     def stage(self, stage_id: str) -> Stage:
-        """Return stage model that match with an input stage ID.
+        """Return stage instance that exists in this job via passing an input
+        stage ID.
+
+        :raise ValueError: If an input stage ID does not found on this job.
 
         :param stage_id: A stage ID that want to extract from this job.
         :rtype: Stage
@@ -337,6 +347,13 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
+    def check_needs(self, jobs: dict[str, Any]) -> bool:
+        """Return True if job's need exists in an input list of job's ID.
+
+        :rtype: bool
+        """
+        return all(need in jobs for need in self.needs)
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the receive context. The
         result from execution will pass to value of ``strategies`` key.
@@ -360,8 +377,12 @@ class Job(BaseModel):
                         }
                     }
 
+        :raise JobException: If the job's ID does not set and the setting
+            default job ID flag does not set.
+
         :param output: An output context.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         if self.id is None and not config.job_default_id:
@@ -387,8 +408,8 @@ class Job(BaseModel):
         self,
         strategy: DictData,
         params: DictData,
-        run_id: str | None = None,
         *,
+        run_id: str | None = None,
         event: Event | None = None,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
@@ -398,11 +419,15 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
+            The result of this execution will return result with strategy ID
+        that generated from the `gen_id` function with a input strategy value.
+
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
 
-        :param strategy: A metrix strategy value.
-        :param params: A dynamic parameters.
+        :param strategy: A strategy metrix value that use on this execution.
+            This value will pass to the `matrix` key for templating.
+        :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
         :param event: An manger event that pass to the PoolThreadExecutor.
 
@@ -410,6 +435,7 @@ class Job(BaseModel):
         """
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
+        rs: Result = Result(run_id=run_id)
 
         # PARAGRAPH:
         #
@@ -430,21 +456,27 @@ class Job(BaseModel):
         for stage in self.stages:
 
             if stage.is_skipped(params=context):
-                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
+                logger.info(
+                    f"({cut_id(run_id)}) [JOB]: Skip stage: {stage.iden!r}"
+                )
                 continue
 
             logger.info(
-                f"({run_id}) [JOB]: Start execute the stage: {stage.iden!r}"
+                f"({cut_id(run_id)}) [JOB]: Execute stage: {stage.iden!r}"
             )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({run_id}) [JOB]: Matrix: {strategy}")
+                logger.info(f"({cut_id(run_id)}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
             if event and event.is_set():
-                return Result(
+                error_msg: str = (
+                    "Job strategy was canceled from event that had set before "
+                    "strategy execution."
+                )
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -453,18 +485,12 @@ class Job(BaseModel):
                             #   it will not filter function object from context.
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
+                            #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
-                            "error_message": (
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
+                            "error": JobException(error_msg),
+                            "error_message": error_msg,
                         },
                     },
-                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -492,14 +518,14 @@ class Job(BaseModel):
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -509,13 +535,12 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
-                    run_id=run_id,
                 )
 
-            # NOTE: Remove the current stage object.
+            # NOTE: Remove the current stage object for saving memory.
             del stage
 
-        return Result(
+        return rs.catch(
             status=0,
             context={
                 strategy_id: {
@@ -523,7 +548,6 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
-            run_id=run_id,
         )
 
     def execute(self, params: DictData, run_id: str | None = None) -> Result:
@@ -542,7 +566,8 @@ class Job(BaseModel):
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy.
+        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
@@ -566,6 +591,7 @@ class Job(BaseModel):
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
         ) as executor:
+
             futures: list[Future] = [
                 executor.submit(
                     self.execute_strategy,
@@ -577,11 +603,8 @@ class Job(BaseModel):
                 for strategy in self.strategy.make()
             ]
 
-            # NOTE: Dynamic catching futures object with fail-fast flag.
             return (
-                self.__catch_fail_fast(
-                    event=event, futures=futures, run_id=run_id
-                )
+                self.__catch_fail_fast(event, futures=futures, run_id=run_id)
                 if self.strategy.fail_fast
                 else self.__catch_all_completed(futures=futures, run_id=run_id)
             )
@@ -593,36 +616,32 @@ class Job(BaseModel):
         run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop all not done futures if it receive the first exception from all
-        running futures.
+        stop and set event on all not done futures if it receive the first
+        exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the
-            observing thread/process.
+            observing multithreading.
         :param futures: A list of futures.
         :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
         # NOTE: Get results from a collection of tasks with a timeout that has
         #   the first exception.
         done, not_done = wait(
-            futures,
-            timeout=timeout,
-            return_when=FIRST_EXCEPTION,
+            futures, timeout=timeout, return_when=FIRST_EXCEPTION
         )
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({cut_id(run_id)}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -635,11 +654,13 @@ class Job(BaseModel):
 
         future: Future
         for future in done:
+
+            # NOTE: Handle the first exception from feature
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({run_id}) [JOB]: One stage failed with: "
-                    f"{future.exception()}, shutting down this future."
+                    f"({cut_id(run_id)}) [JOB]: Fail-fast catching:\n\t"
+                    f"{future.exception()}"
                 )
                 context.update(
                     {
@@ -650,7 +671,7 @@ class Job(BaseModel):
                 continue
 
             # NOTE: Update the result context to main job context.
-            context.update(future.result(timeout=result_timeout).context)
+            context.update(future.result().context)
 
         return rs_final.catch(status=status, context=context)
 
@@ -660,45 +681,27 @@ class Job(BaseModel):
         run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with all-completed mode.
 
-        :param futures: A list of futures that want to catch all completed
-            result.
+        :param futures: A list of futures.
         :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
+
         for future in as_completed(futures, timeout=timeout):
             try:
-                context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:  # pragma: no cov
-                status = 1
-                logger.warning(
-                    f"({run_id}) [JOB]: Task is hanging. Attempting to "
-                    f"kill."
-                )
-                future.cancel()
-                time.sleep(0.1)
-
-                stmt: str = (
-                    "Failed to cancel the task."
-                    if not future.cancelled()
-                    else "Task canceled successfully."
-                )
-                logger.warning(f"({run_id}) [JOB]: {stmt}")
+                context.update(future.result().context)
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({run_id}) [JOB]: Get stage exception with "
-                    f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
-                    f"{err}"
+                    f"({cut_id(run_id)}) [JOB]: All-completed catching:\n\t"
+                    f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(
                     {
@@ -706,4 +709,5 @@ class Job(BaseModel):
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
                 )
+
         return rs_final.catch(status=status, context=context)

ddeutil/workflow/on.py

@@ -189,7 +189,10 @@ class On(BaseModel):
         date that given from input.
         """
         runner: CronRunner = self.generate(start=start)
+
+        # NOTE: ship the next date of runner object that create from start.
         _ = runner.next
+
         return runner
 
 

ddeutil/workflow/stage.py

@@ -55,6 +55,7 @@ from .utils import (
     Registry,
     Result,
     TagFunc,
+    cut_id,
     gen_id,
     make_exec,
     make_registry,
@@ -124,13 +125,16 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 run_id: str = gen_id(self.name + (self.id or ""), unique=True)
                 kwargs["run_id"] = run_id
 
+            rs_raise: Result = Result(status=1, run_id=run_id)
+
             try:
                 # NOTE: Start calling origin function with a passing args.
                 return func(self, *args, **kwargs)
             except Exception as err:
                 # NOTE: Start catching error from the stage execution.
                 logger.error(
-                    f"({run_id}) [STAGE]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
+                    f"{err}"
                 )
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
@@ -147,13 +151,12 @@ def handler_result(message: str | None = None) -> DecoratorResult:
 
                 # NOTE: Catching exception error object to result with
                 #   error_message and error keys.
-                return Result(
+                return rs_raise.catch(
                     status=1,
                     context={
                         "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                    run_id=run_id,
                 )
 
         return wrapped
@@ -339,7 +342,7 @@ class EmptyStage(BaseStage):
         :rtype: Result
         """
         logger.info(
-            f"({run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
+            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
@@ -393,7 +396,9 @@ class BashStage(BaseStage):
         f_name: str = f"{run_id}.sh"
         f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
 
-        logger.debug(f"({run_id}) [STAGE]: Start create `{f_name}` file.")
+        logger.debug(
+            f"({cut_id(run_id)}) [STAGE]: Start create `{f_name}` file."
+        )
 
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: write header of `.sh` file
@@ -425,7 +430,7 @@ class BashStage(BaseStage):
         """
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({run_id}) [STAGE]: Shell-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
             bash=bash, env=param2template(self.env, params), run_id=run_id
         ) as sh:
@@ -496,6 +501,7 @@ class PyStage(BaseStage):
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         # NOTE: The output will fileter unnecessary keys from locals.
@@ -534,7 +540,7 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({run_id}) [STAGE]: Py-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is vary dangerous. So, it
         #   should us the re module to validate exec-string before running.
@@ -638,8 +644,7 @@ class HookStage(BaseStage):
 
         :rtype: Result
         """
-        t_func_hook: str = param2template(self.uses, params)
-        t_func: TagFunc = extract_hook(t_func_hook)()
+        t_func: TagFunc = extract_hook(param2template(self.uses, params))()
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
@@ -660,7 +665,8 @@ class HookStage(BaseStage):
                 args[k] = args.pop(k.removeprefix("_"))
 
         logger.info(
-            f"({run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
+            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
+            f"{t_func.name}@{t_func.tag}"
         )
         rs: DictData = t_func(**param2template(args, params))
 
@@ -716,7 +722,9 @@ class TriggerStage(BaseStage):
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
         wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(f"({run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
+        logger.info(
+            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
+        )
         return wf.execute(
             params=param2template(self.params, params),
             run_id=run_id,

ddeutil/workflow/utils.py

@@ -430,7 +430,7 @@ class Result:
         return self
 
 
-def make_exec(path: str | Path) -> None:  # pragma: no cov
+def make_exec(path: str | Path) -> None:
     """Change mode of file to be executable file.
 
     :param path: A file path that want to make executable permission.
@@ -451,7 +451,9 @@ FILTERS: dict[str, callable] = {  # pragma: no cov
 
 
 class FilterFunc(Protocol):
-    """Tag Function Protocol"""
+    """Tag Function Protocol. This protocol that use to represent any callable
+    object that able to access the name attribute.
+    """
 
     name: str
 
@@ -814,3 +816,17 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
 
 def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
     return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
+
+
+def cut_id(run_id: str, *, num: int = 6):
+    """Cutting running ID with length.
+
+    Example:
+        >>> cut_id(run_id='668931127320241228100331254567')
+        '254567'
+
+    :param run_id:
+    :param num:
+    :return:
+    """
+    return run_id[-num:]