ddeutil-workflow 0.0.26.post1__py3-none-any.whl → 0.0.28__py3-none-any.whl

ddeutil/workflow/job.py

@@ -38,13 +38,13 @@ from .exceptions import (
 )
 from .result import Result
 from .stage import Stage
+from .templates import has_template
 from .utils import (
     cross_product,
     cut_id,
     dash2underscore,
     filter_func,
     gen_id,
-    has_template,
 )
 
 logger = get_logger("ddeutil.workflow")
@@ -83,7 +83,7 @@ def make(
     if len(matrix) == 0:
         return [{}]
 
-    # NOTE: Remove matrix that exists on the exclude.
+    # NOTE: Remove matrix that exists on the excluded.
     final: list[DictStr] = []
     for r in cross_product(matrix=matrix):
         if any(
@@ -101,7 +101,7 @@ def make(
     add: list[DictStr] = []
     for inc in include:
         # VALIDATE:
-        #   Validate any key in include list should be a subset of some one
+        #   Validate any key in include list should be a subset of someone
         #   in matrix.
         if all(not (set(inc.keys()) <= set(m.keys())) for m in final):
             raise ValueError(
@@ -128,9 +128,9 @@ class Strategy(BaseModel):
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
-    any model object. The propose of this model is generate metrix result that
-    comming from combination logic with any matrix values for running it with
-    parallelism.
+    any model object. The objective of this model is generating metrix result
+    that comming from combination logic with any matrix values for running it
+    with parallelism.
 
         [1, 2, 3] x [a, b] --> [1a], [1b], [2a], [2b], [3a], [3b]
 
@@ -180,7 +180,7 @@ class Strategy(BaseModel):
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
 
-        :param values: A parsing values to this models
+        :param values: A parsing values to these models
         :rtype: DictData
         """
         dash2underscore("max-parallel", values)
@@ -226,7 +226,7 @@ class Job(BaseModel):
     """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
-    you pass matrix mapping and it able to generate, you will see it running
+    you pass matrix mapping, and it is able to generate, you will see it running
     with loop of matrix values.
 
     Data Validate:
@@ -355,7 +355,7 @@ class Job(BaseModel):
         return all(need in jobs for need in self.needs)
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to the receive context. The
+        """Set an outputs from execution process to the received context. The
         result from execution will pass to value of ``strategies`` key.
 
             For example of setting output method, If you receive execute output
@@ -420,7 +420,7 @@ class Job(BaseModel):
         strategy and return with context of this strategy data.
 
             The result of this execution will return result with strategy ID
-        that generated from the `gen_id` function with a input strategy value.
+        that generated from the `gen_id` function with an input strategy value.
 
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
@@ -429,7 +429,7 @@ class Job(BaseModel):
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
-        :param event: An manger event that pass to the PoolThreadExecutor.
+        :param event: An event manager that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
@@ -496,7 +496,7 @@ class Job(BaseModel):
             # PARAGRAPH:
             #
             #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer and it was changed when I action
+            #   reference memory pointer, and it was changed when I action
             #   anything like update or re-construct this.
             #
             #       ... params |= stage.execute(params=params)
@@ -513,7 +513,9 @@ class Job(BaseModel):
             #
             try:
                 stage.set_outputs(
-                    stage.execute(params=context, run_id=run_id).context,
+                    stage.handler_execute(
+                        params=context, run_id=run_id
+                    ).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
@@ -566,7 +568,7 @@ class Job(BaseModel):
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
@@ -585,8 +587,7 @@ class Job(BaseModel):
         event: Event = Event()
 
         # IMPORTANT: Start running strategy execution by multithreading because
-        #   it will running by strategy values without waiting previous
-        #   execution.
+        #   it will run by strategy values without waiting previous execution.
         with ThreadPoolExecutor(
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
@@ -618,7 +619,7 @@ class Job(BaseModel):
         timeout: int = 1800,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop and set event on all not done futures if it receive the first
+        stop and set event on all not done futures if it receives the first
         exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the

ddeutil/workflow/params.py

@@ -75,7 +75,7 @@ class DatetimeParam(DefaultParam):
     default: datetime = Field(default_factory=get_dt_now)
 
     def receive(self, value: str | datetime | date | None = None) -> datetime:
-        """Receive value that match with datetime. If a input value pass with
+        """Receive value that match with datetime. If an input value pass with
         None, it will use default value instead.
 
         :param value: A value that want to validate with datetime parameter
@@ -98,7 +98,7 @@ class DatetimeParam(DefaultParam):
             return datetime.fromisoformat(value)
         except ValueError:
             raise ParamValueException(
-                f"Invalid isoformat string: {value!r}"
+                f"Invalid the ISO format string: {value!r}"
             ) from None
 
 
@@ -158,7 +158,7 @@ class ChoiceParam(BaseParam):
         :rtype: str
         """
         # NOTE:
-        #   Return the first value in options if does not pass any input value
+        #   Return the first value in options if it does not pass any input value
         if value is None:
             return self.options[0]
         if value not in self.options:

ddeutil/workflow/result.py

@@ -37,8 +37,8 @@ class Result:
 
     @model_validator(mode="after")
     def __prepare_run_id(self) -> Self:
-        """Prepare running ID which use default ID if it initialize at the first
-        time
+        """Prepare running ID which use default ID if it initializes at the
+        first time.
 
         :rtype: Self
         """
@@ -84,7 +84,7 @@ class Result:
 
     def receive_jobs(self, result: Result) -> Self:
         """Receive context from another result object that use on the workflow
-        execution which create a ``jobs`` keys on the context if it do not
+        execution which create a ``jobs`` keys on the context if it does not
         exist.
 
         :rtype: Self

ddeutil/workflow/scheduler.py

@@ -149,7 +149,7 @@ class WorkflowSchedule(BaseModel):
     @field_validator("on", mode="after")
     def __on_no_dup__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
-        contain every minute value, it should has only one on value.
+        contains every minute value, it should have only one on value.
 
         :rtype: list[On]
         """
@@ -195,8 +195,8 @@ class WorkflowSchedule(BaseModel):
         wf: Workflow = Workflow.from_loader(self.name, externals=extras)
         wf_queue: WorkflowQueue = queue[self.alias]
 
-        # IMPORTANT: Create the default 'on' value if it does not passing
-        #   the on field to the Schedule object.
+        # IMPORTANT: Create the default 'on' value if it does not pass the `on`
+        #   field to the Schedule object.
         ons: list[On] = self.on or wf.on.copy()
 
         for on in ons:
@@ -223,7 +223,7 @@ class WorkflowSchedule(BaseModel):
 class Schedule(BaseModel):
     """Schedule Pydantic model that use to run with any scheduler package.
 
-        It does not equal the on value in Workflow model but it use same logic
+        It does not equal the on value in Workflow model, but it uses same logic
     to running release date with crontab interval.
     """
 
@@ -368,7 +368,7 @@ def schedule_task(
     :param stop: A stop datetime object that force stop running scheduler.
     :param queue: A mapping of alias name and WorkflowQueue object.
     :param threads: A mapping of alias name and Thread object.
-    :param log: A log class that want to making log object.
+    :param log: A log class that want to make log object.
 
     :rtype: CancelJob | None
     """
@@ -449,7 +449,7 @@ def schedule_task(
 
 
 def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
-    """Monitoring function that running every five minute for track long running
+    """Monitoring function that running every five minute for track long-running
     thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
@@ -479,7 +479,7 @@ def schedule_control(
     """Scheduler control function that running every minute.
 
     :param schedules: A list of workflow names that want to schedule running.
-    :param stop: An datetime value that use to stop running schedule.
+    :param stop: A datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
     :param log:
 
@@ -554,7 +554,7 @@ def schedule_control(
         scheduler.run_pending()
         time.sleep(1)
 
-        # NOTE: Break the scheduler when the control job does not exists.
+        # NOTE: Break the scheduler when the control job does not exist.
         if not scheduler.get_jobs("control"):
             scheduler.clear("monitor")
 
@@ -585,7 +585,7 @@ def schedule_runner(
 
     :param stop: A stop datetime object that force stop running scheduler.
     :param externals:
-    :param excluded: A list of schedule name that want to excluded from finding.
+    :param excluded: A list of schedule name that want to exclude from finding.
 
     :rtype: list[str]
 

ddeutil/workflow/stage.py

@@ -5,12 +5,12 @@
 # ------------------------------------------------------------------------------
 """Stage Model that use for getting stage data template from the Job Model.
 The stage handle the minimize task that run in some thread (same thread at
-its job owner) that mean it is the lowest executor of a workflow workflow that
-can tracking logs.
+its job owner) that mean it is the lowest executor of a workflow that can
+tracking logs.
 
     The output of stage execution only return 0 status because I do not want to
 handle stage error on this stage model. I think stage model should have a lot of
-usecase and it does not worry when I want to create a new one.
+use-case, and it does not worry when I want to create a new one.
 
     Execution   --> Ok      --> Result with 0
 
@@ -31,41 +31,28 @@ import time
 import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from dataclasses import dataclass
-from functools import wraps
 from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
-from typing import Callable, Optional, Union
-
-try:
-    from typing import ParamSpec
-except ImportError:
-    from typing_extensions import ParamSpec
+from typing import Optional, Union
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, Re, TupleStr
+from .__types import DictData, DictStr, TupleStr
 from .conf import config, get_logger
 from .exceptions import StageException
+from .hook import TagFunc, extract_hook
 from .result import Result
+from .templates import not_in_template, param2template
 from .utils import (
-    Registry,
-    TagFunc,
     cut_id,
     gen_id,
     make_exec,
-    make_registry,
-    not_in_template,
-    param2template,
 )
 
-P = ParamSpec("P")
-ReturnResult = Callable[P, Result]
-DecoratorResult = Callable[[ReturnResult], ReturnResult]
 logger = get_logger("ddeutil.workflow")
 
 
@@ -76,94 +63,9 @@ __all__: TupleStr = (
     "HookStage",
     "TriggerStage",
     "Stage",
-    "extract_hook",
 )
 
 
-def handler_result(message: str | None = None) -> DecoratorResult:
-    """Decorator function for handler result from the stage execution. This
-    function should to use with execution method only.
-
-        This stage exception handler still use ok-error concept but it allow
-    you force catching an output result with error message by specific
-    environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
-
-        Execution   --> Ok      --> Result
-                                    |-status: 0
-                                    |-context:
-                                        |-outputs: ...
-
-                    --> Error   --> Result (if env var was set)
-                                    |-status: 1
-                                    |-context:
-                                        |-error: ...
-                                        |-error_message: ...
-
-                    --> Error   --> Raise StageException(...)
-
-        On the last step, it will set the running ID on a return result object
-    from current stage ID before release the final result.
-
-    :param message: A message that want to add at prefix of exception statement.
-    :type message: str | None (Default=None)
-    :rtype: Callable[P, Result]
-    """
-    # NOTE: The prefix message string that want to add on the first exception
-    #   message dialog.
-    #
-    #       >>> ValueError: {message}
-    #       ...     raise value error from the stage execution process.
-    #
-    message: str = message or ""
-
-    def decorator(func: ReturnResult) -> ReturnResult:
-
-        @wraps(func)
-        def wrapped(self: Stage, *args, **kwargs):
-
-            if not (run_id := kwargs.get("run_id")):
-                run_id: str = gen_id(self.name + (self.id or ""), unique=True)
-                kwargs["run_id"] = run_id
-
-            rs_raise: Result = Result(status=1, run_id=run_id)
-
-            try:
-                # NOTE: Start calling origin function with a passing args.
-                return func(self, *args, **kwargs)
-            except Exception as err:
-                # NOTE: Start catching error from the stage execution.
-                logger.error(
-                    f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
-                    f"{err}"
-                )
-                if config.stage_raise_error:
-                    # NOTE: If error that raise from stage execution course by
-                    #   itself, it will return that error with previous
-                    #   dependency.
-                    if isinstance(err, StageException):
-                        raise StageException(
-                            f"{self.__class__.__name__}: {message}\n\t{err}"
-                        ) from err
-                    raise StageException(
-                        f"{self.__class__.__name__}: {message}\n\t"
-                        f"{err.__class__.__name__}: {err}"
-                    ) from None
-
-                # NOTE: Catching exception error object to result with
-                #   error_message and error keys.
-                return rs_raise.catch(
-                    status=1,
-                    context={
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
-                )
-
-        return wrapped
-
-    return decorator
-
-
 class BaseStage(BaseModel, ABC):
     """Base Stage Model that keep only id and name fields for the stage
     metadata. If you want to implement any custom stage, you can use this class
@@ -230,8 +132,74 @@ class BaseStage(BaseModel, ABC):
         """
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
+    def handler_execute(
+        self, params: DictData, *, run_id: str | None = None
+    ) -> Result:
+        """Handler result from the stage execution.
+
+            This stage exception handler still use ok-error concept, but it
+        allows you force catching an output result with error message by
+        specific environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
+
+            Execution   --> Ok      --> Result
+                                        |-status: 0
+                                        |-context:
+                                            |-outputs: ...
+
+                        --> Error   --> Result (if env var was set)
+                                        |-status: 1
+                                        |-context:
+                                            |-error: ...
+                                            |-error_message: ...
+
+                        --> Error   --> Raise StageException(...)
+
+            On the last step, it will set the running ID on a return result object
+        from current stage ID before release the final result.
+
+        :param params: A parameter data that want to use in this execution.
+        :param run_id: A running stage ID for this execution.
+
+        :rtype: Result
+        """
+        if not run_id:
+            run_id: str = gen_id(self.name + (self.id or ""), unique=True)
+
+        rs_raise: Result = Result(status=1, run_id=run_id)
+        try:
+            # NOTE: Start calling origin function with a passing args.
+            return self.execute(params, run_id=run_id)
+        except Exception as err:
+            # NOTE: Start catching error from the stage execution.
+            logger.error(
+                f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
+                f"{err}"
+            )
+            if config.stage_raise_error:
+                # NOTE: If error that raise from stage execution course by
+                #   itself, it will return that error with previous
+                #   dependency.
+                if isinstance(err, StageException):
+                    raise StageException(
+                        f"{self.__class__.__name__}: \n\t{err}"
+                    ) from err
+                raise StageException(
+                    f"{self.__class__.__name__}: \n\t"
+                    f"{err.__class__.__name__}: {err}"
+                ) from None
+
+            # NOTE: Catching exception error object to result with
+            #   error_message and error keys.
+            return rs_raise.catch(
+                status=1,
+                context={
+                    "error": err,
+                    "error_message": f"{err.__class__.__name__}: {err}",
+                },
+            )
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to the receive context. The
+        """Set an outputs from execution process to the received context. The
         result from execution will pass to value of ``outputs`` key.
 
             For example of setting output method, If you receive execute output
@@ -248,7 +216,7 @@ class BaseStage(BaseModel, ABC):
                         }
                     }
 
-        :param output: A output data that want to extract to an output key.
+        :param output: An output data that want to extract to an output key.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
@@ -293,8 +261,9 @@ class BaseStage(BaseModel, ABC):
         params: DictData = {} if params is None else params
 
         try:
-            # WARNING: The eval build-in function is vary dangerous. So, it
-            #   should us the re module to validate eval-string before running.
+            # WARNING: The eval build-in function is very dangerous. So, it
+            #   should use the `re` module to validate eval-string before
+            #   running.
             rs: bool = eval(
                 param2template(self.condition, params), globals() | params, {}
             )
@@ -326,7 +295,6 @@ class EmptyStage(BaseStage):
         ge=0,
     )
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
@@ -357,7 +325,7 @@ class EmptyStage(BaseStage):
 
 class BashStage(BaseStage):
     """Bash execution stage that execute bash script on the current OS.
-    That mean if your current OS is Windows, it will running bash in the WSL.
+    If your current OS is Windows, it will run on the bash in the WSL.
 
         I get some limitation when I run shell statement with the built-in
     supprocess package. It does not good enough to use multiline statement.
@@ -423,7 +391,6 @@ class BashStage(BaseStage):
         # Note: Remove .sh file that use to run bash.
         Path(f"./{f_name}").unlink()
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
@@ -504,7 +471,7 @@ class PyStage(BaseStage):
         """Override set an outputs method for the Python execution process that
         extract output from all the locals values.
 
-        :param output: A output data that want to extract to an output key.
+        :param output: An output data that want to extract to an output key.
         :param to: A context data that want to add output result.
 
         :rtype: DictData
@@ -525,7 +492,6 @@ class PyStage(BaseStage):
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
@@ -547,8 +513,8 @@ class PyStage(BaseStage):
         # NOTE: Start exec the run statement.
         logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
 
-        # WARNING: The exec build-in function is vary dangerous. So, it
-        #   should us the re module to validate exec-string before running.
+        # WARNING: The exec build-in function is very dangerous. So, it
+        #   should use the re module to validate exec-string before running.
         exec(run, _globals, lc)
 
         return Result(
@@ -558,53 +524,6 @@ class PyStage(BaseStage):
         )
 
 
-@dataclass(frozen=True)
-class HookSearchData:
-    """Hook Search dataclass that use for receive regular expression grouping
-    dict from searching hook string value.
-    """
-
-    path: str
-    func: str
-    tag: str
-
-
-def extract_hook(hook: str) -> Callable[[], TagFunc]:
-    """Extract Hook function from string value to hook partial function that
-    does run it at runtime.
-
-    :raise NotImplementedError: When the searching hook's function result does
-        not exist in the registry.
-    :raise NotImplementedError: When the searching hook's tag result does not
-        exists in the registry with its function key.
-
-    :param hook: A hook value that able to match with Task regex.
-    :rtype: Callable[[], TagFunc]
-    """
-    if not (found := Re.RE_TASK_FMT.search(hook)):
-        raise ValueError(
-            f"Hook {hook!r} does not match with hook format regex."
-        )
-
-    # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
-    hook: HookSearchData = HookSearchData(**found.groupdict())
-
-    # NOTE: Registry object should implement on this package only.
-    rgt: dict[str, Registry] = make_registry(f"{hook.path}")
-    if hook.func not in rgt:
-        raise NotImplementedError(
-            f"``REGISTER-MODULES.{hook.path}.registries`` does not "
-            f"implement registry: {hook.func!r}."
-        )
-
-    if hook.tag not in rgt[hook.func]:
-        raise NotImplementedError(
-            f"tag: {hook.tag!r} does not found on registry func: "
-            f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
-        )
-    return rgt[hook.func][hook.tag]
-
-
 class HookStage(BaseStage):
     """Hook executor that hook the Python function from registry with tag
     decorator function in ``utils`` module and run it with input arguments.
@@ -612,7 +531,7 @@ class HookStage(BaseStage):
         This stage is different with PyStage because the PyStage is just calling
     a Python statement with the ``eval`` and pass that locale before eval that
     statement. So, you can create your function complexly that you can for your
-    propose to invoked by this stage object.
+    objective to invoked by this stage object.
 
     Data Validate:
         >>> stage = {
@@ -633,7 +552,6 @@ class HookStage(BaseStage):
         alias="with",
     )
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Hook function that already in the hook registry.
 
@@ -664,7 +582,7 @@ class HookStage(BaseStage):
                 f"Necessary params, ({', '.join(ips.parameters.keys())}, ), "
                 f"does not set to args"
             )
-        # NOTE: add '_' prefix if it want to use.
+        # NOTE: add '_' prefix if it wants to use.
         for k in ips.parameters:
             if k.removeprefix("_") in args:
                 args[k] = args.pop(k.removeprefix("_"))
@@ -686,7 +604,7 @@ class HookStage(BaseStage):
 
 
 class TriggerStage(BaseStage):
-    """Trigger Workflow execution stage that execute another workflow. This this
+    """Trigger Workflow execution stage that execute another workflow. This
     the core stage that allow you to create the reusable workflow object or
     dynamic parameters workflow for common usecase.
 
@@ -708,9 +626,8 @@ class TriggerStage(BaseStage):
         description="A parameter that want to pass to workflow execution.",
     )
 
-    @handler_result("Raise from TriggerStage")
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
-        """Trigger another workflow execution. It will waiting the trigger
+        """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
@@ -739,7 +656,7 @@ class TriggerStage(BaseStage):
 # NOTE:
 #   An order of parsing stage model on the Job model with ``stages`` field.
 #   From the current build-in stages, they do not have stage that have the same
-#   fields that be cause of parsing on the Job's stages key.
+#   fields that because of parsing on the Job's stages key.
 #
 Stage = Union[
     PyStage,