ddeutil-workflow 0.0.21__tar.gz → 0.0.22__tar.gz

{ddeutil_workflow-0.0.21/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.21
+Version: 0.0.22
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -196,6 +196,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
 | `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_ON_PER_WORKFLOW`     | Core      | 5                                |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

{ddeutil_workflow-0.0.21 → ddeutil_workflow-0.0.22}/README.md

@@ -163,6 +163,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
 | `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_ON_PER_WORKFLOW`     | Core      | 5                                |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

ddeutil_workflow-0.0.22/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.22"

{ddeutil_workflow-0.0.21 → ddeutil_workflow-0.0.22}/src/ddeutil/workflow/conf.py

@@ -106,6 +106,9 @@ class Config:
     max_poking_pool_worker: int = int(
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )
+    max_on_per_workflow: int = int(
+        env("WORKFLOW_CORE_MAX_ON_PER_WORKFLOW", "5")
+    )
 
     # NOTE: Schedule App
     max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
@@ -462,6 +465,7 @@ class FileLog(BaseLog):
 
         :param excluded: An excluded list of key name that want to pass in the
             model_dump method.
+
         :rtype: Self
         """
         # NOTE: Check environ variable was set for real writing.

{ddeutil_workflow-0.0.21 → ddeutil_workflow-0.0.22}/src/ddeutil/workflow/job.py

@@ -22,7 +22,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
@@ -40,6 +40,7 @@ from .stage import Stage
 from .utils import (
     Result,
     cross_product,
+    cut_id,
     dash2underscore,
     filter_func,
     gen_id,
@@ -346,6 +347,13 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
+    def check_needs(self, jobs: dict[str, Any]) -> bool:
+        """Return True if job's need exists in an input list of job's ID.
+
+        :rtype: bool
+        """
+        return all(need in jobs for need in self.needs)
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the receive context. The
         result from execution will pass to value of ``strategies`` key.
@@ -427,6 +435,7 @@ class Job(BaseModel):
         """
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
+        rs: Result = Result(run_id=run_id)
 
         # PARAGRAPH:
         #
@@ -447,14 +456,18 @@ class Job(BaseModel):
         for stage in self.stages:
 
             if stage.is_skipped(params=context):
-                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
+                logger.info(
+                    f"({cut_id(run_id)}) [JOB]: Skip stage: {stage.iden!r}"
+                )
                 continue
 
-            logger.info(f"({run_id}) [JOB]: Execute stage: {stage.iden!r}")
+            logger.info(
+                f"({cut_id(run_id)}) [JOB]: Execute stage: {stage.iden!r}"
+            )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({run_id}) [JOB]: ... Matrix: {strategy}")
+                logger.info(f"({cut_id(run_id)}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -463,7 +476,7 @@ class Job(BaseModel):
                     "Job strategy was canceled from event that had set before "
                     "strategy execution."
                 )
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -478,7 +491,6 @@ class Job(BaseModel):
                             "error_message": error_msg,
                         },
                     },
-                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -506,14 +518,14 @@ class Job(BaseModel):
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -523,13 +535,12 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
-                    run_id=run_id,
                 )
 
             # NOTE: Remove the current stage object for saving memory.
             del stage
 
-        return Result(
+        return rs.catch(
             status=0,
             context={
                 strategy_id: {
@@ -537,7 +548,6 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
-            run_id=run_id,
         )
 
     def execute(self, params: DictData, run_id: str | None = None) -> Result:
@@ -619,7 +629,7 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
@@ -631,7 +641,7 @@ class Job(BaseModel):
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({cut_id(run_id)}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -649,7 +659,7 @@ class Job(BaseModel):
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({run_id}) [JOB]: Fail-fast catching:\n\t"
+                    f"({cut_id(run_id)}) [JOB]: Fail-fast catching:\n\t"
                     f"{future.exception()}"
                 )
                 context.update(
@@ -680,7 +690,7 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
@@ -690,7 +700,7 @@ class Job(BaseModel):
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({run_id}) [JOB]: All-completed catching:\n\t"
+                    f"({cut_id(run_id)}) [JOB]: All-completed catching:\n\t"
                     f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(

{ddeutil_workflow-0.0.21 → ddeutil_workflow-0.0.22}/src/ddeutil/workflow/on.py

@@ -189,7 +189,10 @@ class On(BaseModel):
         date that given from input.
         """
         runner: CronRunner = self.generate(start=start)
+
+        # NOTE: ship the next date of runner object that create from start.
         _ = runner.next
+
         return runner
 
 

{ddeutil_workflow-0.0.21 → ddeutil_workflow-0.0.22}/src/ddeutil/workflow/stage.py

@@ -55,6 +55,7 @@ from .utils import (
     Registry,
     Result,
     TagFunc,
+    cut_id,
     gen_id,
     make_exec,
     make_registry,
@@ -124,13 +125,16 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 run_id: str = gen_id(self.name + (self.id or ""), unique=True)
                 kwargs["run_id"] = run_id
 
+            rs_raise: Result = Result(status=1, run_id=run_id)
+
             try:
                 # NOTE: Start calling origin function with a passing args.
                 return func(self, *args, **kwargs)
             except Exception as err:
                 # NOTE: Start catching error from the stage execution.
                 logger.error(
-                    f"({run_id}) [STAGE]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
+                    f"{err}"
                 )
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
@@ -147,13 +151,12 @@ def handler_result(message: str | None = None) -> DecoratorResult:
 
                 # NOTE: Catching exception error object to result with
                 #   error_message and error keys.
-                return Result(
+                return rs_raise.catch(
                     status=1,
                     context={
                         "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                    run_id=run_id,
                 )
 
         return wrapped
@@ -339,7 +342,7 @@ class EmptyStage(BaseStage):
         :rtype: Result
         """
         logger.info(
-            f"({run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
+            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
@@ -393,7 +396,9 @@ class BashStage(BaseStage):
         f_name: str = f"{run_id}.sh"
         f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
 
-        logger.debug(f"({run_id}) [STAGE]: Start create `{f_name}` file.")
+        logger.debug(
+            f"({cut_id(run_id)}) [STAGE]: Start create `{f_name}` file."
+        )
 
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: write header of `.sh` file
@@ -425,7 +430,7 @@ class BashStage(BaseStage):
         """
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({run_id}) [STAGE]: Shell-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
             bash=bash, env=param2template(self.env, params), run_id=run_id
         ) as sh:
@@ -535,7 +540,7 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({run_id}) [STAGE]: Py-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is vary dangerous. So, it
         #   should us the re module to validate exec-string before running.
@@ -660,7 +665,8 @@ class HookStage(BaseStage):
                 args[k] = args.pop(k.removeprefix("_"))
 
         logger.info(
-            f"({run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
+            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
+            f"{t_func.name}@{t_func.tag}"
         )
         rs: DictData = t_func(**param2template(args, params))
 
@@ -716,7 +722,9 @@ class TriggerStage(BaseStage):
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
         wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(f"({run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
+        logger.info(
+            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
+        )
         return wf.execute(
             params=param2template(self.params, params),
             run_id=run_id,

{ddeutil_workflow-0.0.21 → ddeutil_workflow-0.0.22}/src/ddeutil/workflow/utils.py

@@ -430,7 +430,7 @@ class Result:
         return self
 
 
-def make_exec(path: str | Path) -> None:  # pragma: no cov
+def make_exec(path: str | Path) -> None:
     """Change mode of file to be executable file.
 
     :param path: A file path that want to make executable permission.
@@ -451,7 +451,9 @@ FILTERS: dict[str, callable] = {  # pragma: no cov
 
 
 class FilterFunc(Protocol):
-    """Tag Function Protocol"""
+    """Tag Function Protocol. This protocol that use to represent any callable
+    object that able to access the name attribute.
+    """
 
     name: str
 
@@ -814,3 +816,17 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
 
 def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
     return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
+
+
+def cut_id(run_id: str, *, num: int = 6):
+    """Cutting running ID with length.
+
+    Example:
+        >>> cut_id(run_id='668931127320241228100331254567')
+        '254567'
+
+    :param run_id:
+    :param num:
+    :return:
+    """
+    return run_id[-num:]