ddeutil-workflow 0.0.27__py3-none-any.whl → 0.0.28__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -5,12 +5,12 @@
 # ------------------------------------------------------------------------------
 """Stage Model that use for getting stage data template from the Job Model.
 The stage handle the minimize task that run in some thread (same thread at
-its job owner) that mean it is the lowest executor of a workflow workflow that
-can tracking logs.
+its job owner) that mean it is the lowest executor of a workflow that can
+tracking logs.
 
     The output of stage execution only return 0 status because I do not want to
 handle stage error on this stage model. I think stage model should have a lot of
-usecase and it does not worry when I want to create a new one.
+use-case, and it does not worry when I want to create a new one.
 
     Execution   --> Ok      --> Result with 0
 
@@ -31,17 +31,11 @@ import time
 import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from functools import wraps
 from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
-from typing import Callable, Optional, Union
-
-try:
-    from typing import ParamSpec
-except ImportError:
-    from typing_extensions import ParamSpec
+from typing import Optional, Union
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
@@ -59,9 +53,6 @@ from .utils import (
     make_exec,
 )
 
-P = ParamSpec("P")
-ReturnResult = Callable[P, Result]
-DecoratorResult = Callable[[ReturnResult], ReturnResult]
 logger = get_logger("ddeutil.workflow")
 
 
@@ -75,90 +66,6 @@ __all__: TupleStr = (
 )
 
 
-def handler_result(message: str | None = None) -> DecoratorResult:
-    """Decorator function for handler result from the stage execution. This
-    function should to use with execution method only.
-
-        This stage exception handler still use ok-error concept but it allow
-    you force catching an output result with error message by specific
-    environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
-
-        Execution   --> Ok      --> Result
-                                    |-status: 0
-                                    |-context:
-                                        |-outputs: ...
-
-                    --> Error   --> Result (if env var was set)
-                                    |-status: 1
-                                    |-context:
-                                        |-error: ...
-                                        |-error_message: ...
-
-                    --> Error   --> Raise StageException(...)
-
-        On the last step, it will set the running ID on a return result object
-    from current stage ID before release the final result.
-
-    :param message: A message that want to add at prefix of exception statement.
-    :type message: str | None (Default=None)
-    :rtype: Callable[P, Result]
-    """
-    # NOTE: The prefix message string that want to add on the first exception
-    #   message dialog.
-    #
-    #       >>> ValueError: {message}
-    #       ...     raise value error from the stage execution process.
-    #
-    message: str = message or ""
-
-    def decorator(func: ReturnResult) -> ReturnResult:
-
-        @wraps(func)
-        def wrapped(self: Stage, *args, **kwargs):
-
-            if not (run_id := kwargs.get("run_id")):
-                run_id: str = gen_id(self.name + (self.id or ""), unique=True)
-                kwargs["run_id"] = run_id
-
-            rs_raise: Result = Result(status=1, run_id=run_id)
-
-            try:
-                # NOTE: Start calling origin function with a passing args.
-                return func(self, *args, **kwargs)
-            except Exception as err:
-                # NOTE: Start catching error from the stage execution.
-                logger.error(
-                    f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
-                    f"{err}"
-                )
-                if config.stage_raise_error:
-                    # NOTE: If error that raise from stage execution course by
-                    #   itself, it will return that error with previous
-                    #   dependency.
-                    if isinstance(err, StageException):
-                        raise StageException(
-                            f"{self.__class__.__name__}: {message}\n\t{err}"
-                        ) from err
-                    raise StageException(
-                        f"{self.__class__.__name__}: {message}\n\t"
-                        f"{err.__class__.__name__}: {err}"
-                    ) from None
-
-                # NOTE: Catching exception error object to result with
-                #   error_message and error keys.
-                return rs_raise.catch(
-                    status=1,
-                    context={
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
-                )
-
-        return wrapped
-
-    return decorator
-
-
 class BaseStage(BaseModel, ABC):
     """Base Stage Model that keep only id and name fields for the stage
     metadata. If you want to implement any custom stage, you can use this class
@@ -225,8 +132,74 @@ class BaseStage(BaseModel, ABC):
         """
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
+    def handler_execute(
+        self, params: DictData, *, run_id: str | None = None
+    ) -> Result:
+        """Handler result from the stage execution.
+
+            This stage exception handler still use ok-error concept, but it
+        allows you force catching an output result with error message by
+        specific environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
+
+            Execution   --> Ok      --> Result
+                                        |-status: 0
+                                        |-context:
+                                            |-outputs: ...
+
+                        --> Error   --> Result (if env var was set)
+                                        |-status: 1
+                                        |-context:
+                                            |-error: ...
+                                            |-error_message: ...
+
+                        --> Error   --> Raise StageException(...)
+
+            On the last step, it will set the running ID on a return result object
+        from current stage ID before release the final result.
+
+        :param params: A parameter data that want to use in this execution.
+        :param run_id: A running stage ID for this execution.
+
+        :rtype: Result
+        """
+        if not run_id:
+            run_id: str = gen_id(self.name + (self.id or ""), unique=True)
+
+        rs_raise: Result = Result(status=1, run_id=run_id)
+        try:
+            # NOTE: Start calling origin function with a passing args.
+            return self.execute(params, run_id=run_id)
+        except Exception as err:
+            # NOTE: Start catching error from the stage execution.
+            logger.error(
+                f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
+                f"{err}"
+            )
+            if config.stage_raise_error:
+                # NOTE: If error that raise from stage execution course by
+                #   itself, it will return that error with previous
+                #   dependency.
+                if isinstance(err, StageException):
+                    raise StageException(
+                        f"{self.__class__.__name__}: \n\t{err}"
+                    ) from err
+                raise StageException(
+                    f"{self.__class__.__name__}: \n\t"
+                    f"{err.__class__.__name__}: {err}"
+                ) from None
+
+            # NOTE: Catching exception error object to result with
+            #   error_message and error keys.
+            return rs_raise.catch(
+                status=1,
+                context={
+                    "error": err,
+                    "error_message": f"{err.__class__.__name__}: {err}",
+                },
+            )
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to the receive context. The
+        """Set an outputs from execution process to the received context. The
         result from execution will pass to value of ``outputs`` key.
 
             For example of setting output method, If you receive execute output
@@ -243,7 +216,7 @@ class BaseStage(BaseModel, ABC):
                         }
                     }
 
-        :param output: A output data that want to extract to an output key.
+        :param output: An output data that want to extract to an output key.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
@@ -288,8 +261,9 @@ class BaseStage(BaseModel, ABC):
         params: DictData = {} if params is None else params
 
         try:
-            # WARNING: The eval build-in function is vary dangerous. So, it
-            #   should us the re module to validate eval-string before running.
+            # WARNING: The eval build-in function is very dangerous. So, it
+            #   should use the `re` module to validate eval-string before
+            #   running.
             rs: bool = eval(
                 param2template(self.condition, params), globals() | params, {}
             )
@@ -321,7 +295,6 @@ class EmptyStage(BaseStage):
         ge=0,
     )
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
@@ -352,7 +325,7 @@ class EmptyStage(BaseStage):
 
 class BashStage(BaseStage):
     """Bash execution stage that execute bash script on the current OS.
-    That mean if your current OS is Windows, it will running bash in the WSL.
+    If your current OS is Windows, it will run on the bash in the WSL.
 
         I get some limitation when I run shell statement with the built-in
     supprocess package. It does not good enough to use multiline statement.
@@ -418,7 +391,6 @@ class BashStage(BaseStage):
         # Note: Remove .sh file that use to run bash.
         Path(f"./{f_name}").unlink()
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
@@ -499,7 +471,7 @@ class PyStage(BaseStage):
         """Override set an outputs method for the Python execution process that
         extract output from all the locals values.
 
-        :param output: A output data that want to extract to an output key.
+        :param output: An output data that want to extract to an output key.
         :param to: A context data that want to add output result.
 
         :rtype: DictData
@@ -520,7 +492,6 @@ class PyStage(BaseStage):
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
@@ -542,8 +513,8 @@ class PyStage(BaseStage):
         # NOTE: Start exec the run statement.
         logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
 
-        # WARNING: The exec build-in function is vary dangerous. So, it
-        #   should us the re module to validate exec-string before running.
+        # WARNING: The exec build-in function is very dangerous. So, it
+        #   should use the re module to validate exec-string before running.
         exec(run, _globals, lc)
 
         return Result(
@@ -560,7 +531,7 @@ class HookStage(BaseStage):
         This stage is different with PyStage because the PyStage is just calling
     a Python statement with the ``eval`` and pass that locale before eval that
     statement. So, you can create your function complexly that you can for your
-    propose to invoked by this stage object.
+    objective to invoked by this stage object.
 
     Data Validate:
         >>> stage = {
@@ -581,7 +552,6 @@ class HookStage(BaseStage):
         alias="with",
     )
 
-    @handler_result()
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
         """Execute the Hook function that already in the hook registry.
 
@@ -612,7 +582,7 @@ class HookStage(BaseStage):
                 f"Necessary params, ({', '.join(ips.parameters.keys())}, ), "
                 f"does not set to args"
             )
-        # NOTE: add '_' prefix if it want to use.
+        # NOTE: add '_' prefix if it wants to use.
         for k in ips.parameters:
             if k.removeprefix("_") in args:
                 args[k] = args.pop(k.removeprefix("_"))
@@ -634,7 +604,7 @@ class HookStage(BaseStage):
 
 
 class TriggerStage(BaseStage):
-    """Trigger Workflow execution stage that execute another workflow. This this
+    """Trigger Workflow execution stage that execute another workflow. This
     the core stage that allow you to create the reusable workflow object or
     dynamic parameters workflow for common usecase.
 
@@ -656,9 +626,8 @@ class TriggerStage(BaseStage):
         description="A parameter that want to pass to workflow execution.",
     )
 
-    @handler_result("Raise from TriggerStage")
     def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
-        """Trigger another workflow execution. It will waiting the trigger
+        """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
@@ -687,7 +656,7 @@ class TriggerStage(BaseStage):
 # NOTE:
 #   An order of parsing stage model on the Job model with ``stages`` field.
 #   From the current build-in stages, they do not have stage that have the same
-#   fields that be cause of parsing on the Job's stages key.
+#   fields that because of parsing on the Job's stages key.
 #
 Stage = Union[
     PyStage,

ddeutil/workflow/templates.py

@@ -44,10 +44,10 @@ FILTERS: dict[str, callable] = {  # pragma: no cov
 
 class FilterFunc(Protocol):
     """Tag Function Protocol. This protocol that use to represent any callable
-    object that able to access the name attribute.
+    object that able to access the filter attribute.
     """
 
-    name: str
+    filter: str
 
     def __call__(self, *args, **kwargs): ...  # pragma: no cov
 
@@ -95,6 +95,8 @@ def make_filter_registry() -> dict[str, FilterRegistry]:
             if not hasattr(func, "filter"):
                 continue
 
+            func: FilterFunc
+
             rs[func.filter] = import_string(f"{module}.{fstr}")
 
     rs.update(FILTERS)
@@ -174,7 +176,7 @@ def map_post_filter(
     """Mapping post-filter to value with sequence list of filter function name
     that will get from the filter registry.
 
-    :param value: A string value that want to mapped with filter function.
+    :param value: A string value that want to map with filter function.
     :param post_filter: A list of post-filter function name.
     :param filters: A filter registry.
 
@@ -204,7 +206,7 @@ def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
     """Check value should not pass template with not_in value prefix.
 
     :param value: A value that want to find parameter template prefix.
-    :param not_in: The not in string that use in the `.startswith` function.
+    :param not_in: The not-in string that use in the `.startswith` function.
 
     :rtype: bool
     """
@@ -249,7 +251,7 @@ def str2template(
     with the workflow parameter types that is `str`, `int`, `datetime`, and
     `list`.
 
-    :param value: A string value that want to mapped with an params
+    :param value: A string value that want to map with params
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters:
@@ -273,11 +275,11 @@ def str2template(
         if not hasdot(caller, params):
             raise UtilException(f"The params does not set caller: {caller!r}.")
 
-        # NOTE: from validate step, it guarantee that caller exists in params.
+        # NOTE: from validate step, it guarantees that caller exists in params.
         getter: Any = getdot(caller, params)
 
         # NOTE:
-        #   If type of getter caller is not string type and it does not use to
+        #   If type of getter caller is not string type, and it does not use to
         #   concat other string value, it will return origin value from the
         #   ``getdot`` function.
         if value.replace(found.full, "", 1) == "":
@@ -300,7 +302,7 @@ def param2template(
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
-    :param value: A value that want to mapped with an params
+    :param value: A value that want to map with params
     :param params: A parameter value that getting with matched regular
         expression.
 

ddeutil/workflow/utils.py

@@ -18,18 +18,12 @@ from random import randrange
 from typing import Any, TypeVar
 from zoneinfo import ZoneInfo
 
-try:
-    from typing import ParamSpec
-except ImportError:
-    from typing_extensions import ParamSpec
-
 from ddeutil.core import hash_str
 
 from .__types import DictData, Matrix
 from .conf import config
 
 T = TypeVar("T")
-P = ParamSpec("P")
 
 logger = logging.getLogger("ddeutil.workflow")
 
@@ -86,7 +80,7 @@ def gen_id(
     sensitive: bool = True,
     unique: bool = False,
 ) -> str:
-    """Generate running ID for able to tracking. This generate process use `md5`
+    """Generate running ID for able to tracking. This generates process use `md5`
     algorithm function if ``WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE`` set to
     false. But it will cut this hashing value length to 10 it the setting value
     set to true.
@@ -95,6 +89,7 @@ def gen_id(
     :param sensitive: A flag that convert the value to lower case before hashing
     :param unique: A flag that add timestamp at microsecond level to value
         before hashing.
+
     :rtype: str
     """
     if not isinstance(value, str):
@@ -121,13 +116,13 @@ def make_exec(path: str | Path) -> None:
     f.chmod(f.stat().st_mode | stat.S_IEXEC)
 
 
-def filter_func(value: Any) -> Any:
+def filter_func(value: T) -> T:
     """Filter out an own created function of any value of mapping context by
     replacing it to its function name. If it is built-in function, it does not
     have any changing.
 
     :param value: A value context data that want to filter out function value.
-    :type: The same type of an input ``value``.
+    :type: The same type of input ``value``.
     """
     if isinstance(value, dict):
         return {k: filter_func(value[k]) for k in value}
@@ -135,8 +130,8 @@ def filter_func(value: Any) -> Any:
         return type(value)([filter_func(i) for i in value])
 
     if isfunction(value):
-        # NOTE: If it want to improve to get this function, it able to save to
-        #   some global memory storage.
+        # NOTE: If it wants to improve to get this function, it is able to save
+        # to some global memory storage.
         #   ---
         #   >>> GLOBAL_DICT[value.__name__] = value
         #
@@ -152,6 +147,10 @@ def dash2underscore(
 ) -> DictData:
     """Change key name that has dash to underscore.
 
+    :param key
+    :param values
+    :param fixed
+
     :rtype: DictData
     """
     if key in values:
@@ -162,6 +161,8 @@ def dash2underscore(
 def cross_product(matrix: Matrix) -> Iterator[DictData]:
     """Iterator of products value from matrix.
 
+    :param matrix:
+
     :rtype: Iterator[DictData]
     """
     yield from (
@@ -181,6 +182,11 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         ['A', 'B', 'C']
         ['D', 'E', 'F']
         ['G']
+
+    :param iterable:
+    :param n:
+
+    :rtype: Iterator[Any]
     """
     if n < 1:
         raise ValueError("n must be at least one")
@@ -195,7 +201,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         yield chain((first_el,), chunk_it)
 
 
-def cut_id(run_id: str, *, num: int = 6):
+def cut_id(run_id: str, *, num: int = 6) -> str:
     """Cutting running ID with length.
 
     Example:
@@ -204,6 +210,7 @@ def cut_id(run_id: str, *, num: int = 6):
 
     :param run_id:
     :param num:
-    :return:
+
+    :rtype: str
     """
     return run_id[-num:]

ddeutil/workflow/workflow.py

@@ -138,7 +138,7 @@ class WorkflowQueue:
         """Construct WorkflowQueue object from an input queue value that passing
         with list of datetime or list of WorkflowRelease.
 
-        :raise TypeError: If the type of an input queue does not valid.
+        :raise TypeError: If the type of input queue does not valid.
 
         :rtype: Self
         """
@@ -226,9 +226,9 @@ class WorkflowQueue:
 class Workflow(BaseModel):
     """Workflow Pydantic model.
 
-        This is the main future of this project because it use to be workflow
+        This is the main future of this project because it uses to be workflow
     data for running everywhere that you want or using it to scheduler task in
-    background. It use lightweight coding line from Pydantic Model and enhance
+    background. It uses lightweight coding line from Pydantic Model and enhance
     execute method on it.
     """
 
@@ -317,7 +317,7 @@ class Workflow(BaseModel):
     @model_validator(mode="before")
     def __prepare_model_before__(cls, values: DictData) -> DictData:
         """Prepare the params key in the data model before validating."""
-        # NOTE: Prepare params type if it passing with only type value.
+        # NOTE: Prepare params type if it is passing with only type value.
         if params := values.pop("params", {}):
             values["params"] = {
                 p: (
@@ -341,7 +341,7 @@ class Workflow(BaseModel):
     @field_validator("on", mode="after")
     def __on_no_dup_and_reach_limit__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
-        contain the every minute value more than one value, it will remove to
+        contains the every minute value more than one value, it will remove to
         only one value.
 
         :raise ValueError: If it has some duplicate value.
@@ -359,8 +359,8 @@ class Workflow(BaseModel):
         # WARNING:
         # if '* * * * *' in set_ons and len(set_ons) > 1:
         #     raise ValueError(
-        #         "If it has every minute cronjob on value, it should has only "
-        #         "one value in the on field."
+        #         "If it has every minute cronjob on value, it should have "
+        #         "only one value in the on field."
         #     )
 
         if len(set_ons) > config.max_on_per_workflow:
@@ -372,7 +372,7 @@ class Workflow(BaseModel):
 
     @model_validator(mode="after")
     def __validate_jobs_need__(self) -> Self:
-        """Validate each need job in any jobs should exists.
+        """Validate each need job in any jobs should exist.
 
         :raise WorkflowException: If it has not exists need value in this
             workflow job.
@@ -591,10 +591,10 @@ class Workflow(BaseModel):
         """Generate queue of datetime from the cron runner that initialize from
         the on field. with offset value.
 
-        :param offset: A offset in second unit for time travel.
+        :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to making log object.
+        :param log: A log class that want to make log object.
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.
 
@@ -696,7 +696,7 @@ class Workflow(BaseModel):
             start_date: datetime = current_date
             offset: float = 0
 
-        # NOTE: End date is use to stop generate queue with an input periods
+        # NOTE: End date is using to stop generate queue with an input periods
         #   value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
@@ -812,7 +812,7 @@ class Workflow(BaseModel):
         :param params: A params that was parameterized from workflow execution.
         :param run_id: A workflow running ID for this job execution.
         :param raise_error: A flag that raise error instead catching to result
-            if it get exception from job execution.
+            if it gets exception from job execution.
 
         :rtype: Result
         :return: Return the result object that receive the job execution result
@@ -868,8 +868,8 @@ class Workflow(BaseModel):
         """Execute workflow with passing a dynamic parameters to all jobs that
         included in this workflow model with ``jobs`` field.
 
-            The result of execution process for each jobs and stages on this
-        workflow will keeping in dict which able to catch out with all jobs and
+            The result of execution process for each job and stages on this
+        workflow will keep in dict which able to catch out with all jobs and
         stages by dot annotation.
 
             For example, when I want to use the output from previous stage, I
@@ -907,8 +907,8 @@ class Workflow(BaseModel):
             )
             return rs.catch(status=0, context=params)
 
-        # NOTE: Create a job queue that keep the job that want to running after
-        #   it dependency condition.
+        # NOTE: Create a job queue that keep the job that want to run after
+        #   its dependency condition.
         jq: Queue = Queue()
         for job_id in self.jobs:
             jq.put(job_id)
@@ -967,7 +967,7 @@ class Workflow(BaseModel):
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
-            timeout.
+            time out.
         :param job_queue: A job queue object.
         :param timeout: A second value unit that bounding running time.
         :param thread_timeout: A timeout to waiting all futures complete.
@@ -1064,7 +1064,7 @@ class Workflow(BaseModel):
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
-            timeout.
+            time out.
         :param timeout: A second value unit that bounding running time.
 
         :rtype: DictData
@@ -1090,7 +1090,7 @@ class Workflow(BaseModel):
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
-            #   before release. This job execution process will running until
+            #   before release. This job execution process will run until
             #   done before checking all execution timeout or not.
             #
             #   {
@@ -1182,7 +1182,7 @@ class WorkflowTask:
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to making log object.
+        :param log: A log class that want to make log object.
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.