ddeutil-workflow 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -12,14 +12,16 @@ can tracking logs.
 handle stage error on this stage model. I think stage model should have a lot of
 usecase and it does not worry when I want to create a new one.
 
-    Execution --> Ok    --> Result with 0
-              --> Error --> Raise StageException
+    Execution   --> Ok    --> Result with 0
+                --> Error --> Raise StageException
+
+    On the context I/O that pass to stage object at execute process. The execute
+method receive `{"params": {...}}` for mapping to template.
 """
 from __future__ import annotations
 
 import contextlib
 import inspect
-import os
 import subprocess
 import sys
 import uuid
@@ -38,12 +40,12 @@ try:
 except ImportError:
     from typing_extensions import ParamSpec
 
-from ddeutil.core import str2bool
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Re, TupleStr
+from .conf import config
 from .exceptions import StageException
 from .log import get_logger
 from .utils import (
@@ -62,12 +64,15 @@ logger = get_logger("ddeutil.workflow")
 
 
 __all__: TupleStr = (
-    "Stage",
+    "BaseStage",
     "EmptyStage",
     "BashStage",
     "PyStage",
     "HookStage",
     "TriggerStage",
+    "Stage",
+    "HookSearchData",
+    "extract_hook",
     "handler_result",
 )
 
@@ -76,8 +81,26 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
     """Decorator function for handler result from the stage execution. This
     function should to use with execution method only.
 
+        This stage exception handler still use ok-error concept but it allow
+    you force catching an output result with error message by specific
+    environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
+
+        Execution   --> Ok      --> Result with 0
+                    --> Error   --> Raise StageException
+                                --> Result with 1 (if env var was set)
+
+        On the last step, it will set the running ID on a return result object
+    from current stage ID before release the final result.
+
     :param message: A message that want to add at prefix of exception statement.
+    :rtype: Callable[P, Result]
     """
+    # NOTE: The prefix message string that want to add on the first exception
+    #   message dialog.
+    #
+    #       ... ValueError: {message}
+    #       ...     raise value error from the stage execution process.
+    #
     message: str = message or ""
 
     def decorator(func: Callable[P, Result]) -> Callable[P, Result]:
@@ -92,9 +115,7 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
                 logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
-                if str2bool(
-                    os.getenv("WORKFLOW_CORE_STAGE_RAISE_ERROR", "true")
-                ):
+                if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
                     #   itself, it will return that error with previous
                     #   dependency.
@@ -106,13 +127,16 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
                         f"{self.__class__.__name__}: {message}\n\t"
                         f"{err.__class__.__name__}: {err}"
                     ) from None
-                rs: Result = Result(
+
+                # NOTE: Catching exception error object to result with
+                #   error_message and error keys.
+                return Result(
                     status=1,
                     context={
+                        "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                )
-                return rs.set_run_id(self.run_id)
+                ).set_run_id(self.run_id)
 
         return wrapped
 
@@ -148,10 +172,12 @@ class BaseStage(BaseModel, ABC):
     )
 
     @model_validator(mode="after")
-    def __prepare_running_id(self):
+    def __prepare_running_id(self) -> Self:
         """Prepare stage running ID that use default value of field and this
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
+
+        :rtype: Self
         """
         if self.run_id is None:
             self.run_id = gen_id(self.name + (self.id or ""), unique=True)
@@ -185,16 +211,28 @@ class BaseStage(BaseModel, ABC):
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to an input params.
+        """Set an outputs from execution process to the receive context. The
+        result from execution will pass to value of ``outputs`` key.
+
+            For example of setting output method, If you receive execute output
+        and want to set on the `to` like;
+
+            ... (i)   output: {'foo': bar}
+            ... (ii)  to: {}
+
+        The result of the `to` variable will be;
+
+            ... (iii) to: {
+                            'stages': {
+                                '<stage-id>': {'outputs': {'foo': 'bar'}}
+                            }
+                        }
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
-        if not (
-            self.id
-            or str2bool(os.getenv("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false"))
-        ):
+        if not (self.id or config.stage_default_id):
             logger.debug(
                 f"({self.run_id}) [STAGE]: Output does not set because this "
                 f"stage does not set ID or default stage ID config flag not be "
@@ -206,28 +244,29 @@ class BaseStage(BaseModel, ABC):
         if "stages" not in to:
             to["stages"] = {}
 
-        if self.id:
-            _id: str = param2template(self.id, params=to)
-        else:
-            _id: str = gen_id(param2template(self.name, params=to))
-
-        # NOTE: Set the output to that stage generated ID.
-        logger.debug(
-            f"({self.run_id}) [STAGE]: Set output complete with stage ID: {_id}"
+        # NOTE: If the stage ID did not set, it will use its name instead.
+        _id: str = (
+            param2template(self.id, params=to)
+            if self.id
+            else gen_id(param2template(self.name, params=to))
         )
+
+        # NOTE: Set the output to that stage generated ID with ``outputs`` key.
+        logger.debug(f"({self.run_id}) [STAGE]: Set outputs on: {_id}")
         to["stages"][_id] = {"outputs": output}
         return to
 
     def is_skipped(self, params: DictData | None = None) -> bool:
-        """Return true if condition of this stage do not correct.
+        """Return true if condition of this stage do not correct. This process
+        use build-in eval function to execute the if-condition.
 
         :param params: A parameters that want to pass to condition template.
         :rtype: bool
         """
-        params: DictData = params or {}
         if self.condition is None:
             return False
 
+        params: DictData = {} if params is None else params
         _g: DictData = globals() | params
         try:
             rs: bool = eval(param2template(self.condition, params), _g, {})
@@ -257,10 +296,15 @@ class EmptyStage(BaseStage):
 
     def execute(self, params: DictData) -> Result:
         """Execution method for the Empty stage that do only logging out to
-        stdout.
+        stdout. This method does not use the `handler_result` decorator because
+        it does not get any error from logging function.
+
+            The result context should be empty and do not process anything
+        without calling logging function.
 
         :param params: A context data that want to add output result. But this
             stage does not pass any output.
+        :rtype: Result
         """
         logger.info(
             f"({self.run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
@@ -302,6 +346,10 @@ class BashStage(BaseStage):
         """Return context of prepared bash statement that want to execute. This
         step will write the `.sh` file before giving this file name to context.
         After that, it will auto delete this file automatic.
+
+        :param bash: A bash statement that want to execute.
+        :param env: An environment variable that use on this bash statement.
+        :rtype: Iterator[TupleStr]
         """
         f_name: str = f"{uuid.uuid4()}.sh"
         f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
@@ -348,6 +396,7 @@ class BashStage(BaseStage):
                 text=True,
             )
         if rs.returncode > 0:
+            # NOTE: Prepare stderr message that returning from subprocess.
             err: str = (
                 rs.stderr.encode("utf-8").decode("utf-16")
                 if "\\x00" in rs.stderr
@@ -368,8 +417,11 @@ class BashStage(BaseStage):
 
 
 class PyStage(BaseStage):
-    """Python executor stage that running the Python statement that receive
-    globals nad additional variables.
+    """Python executor stage that running the Python statement with receiving
+    globals and additional variables.
+
+        This stage allow you to use any Python object that exists on the globals
+    such as import your installed package.
 
     Data Validate:
         >>> stage = {
@@ -392,7 +444,8 @@ class PyStage(BaseStage):
     )
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from the Python execution process to an input params.
+        """Override set an outputs method for the Python execution process that
+        extract output from all the locals values.
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
@@ -432,12 +485,13 @@ class PyStage(BaseStage):
         exec(run, _globals, _locals)
 
         return Result(
-            status=0, context={"locals": _locals, "globals": _globals}
+            status=0,
+            context={"locals": _locals, "globals": _globals},
         )
 
 
-@dataclass
-class HookSearch:
+@dataclass(frozen=True)
+class HookSearchData:
     """Hook Search dataclass that use for receive regular expression grouping
     dict from searching hook string value.
     """
@@ -460,7 +514,7 @@ def extract_hook(hook: str) -> Callable[[], TagFunc]:
         )
 
     # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
-    hook: HookSearch = HookSearch(**found.groupdict())
+    hook: HookSearchData = HookSearchData(**found.groupdict())
 
     # NOTE: Registry object should implement on this package only.
     rgt: dict[str, Registry] = make_registry(f"{hook.path}")
@@ -551,7 +605,9 @@ class HookStage(BaseStage):
 
 
 class TriggerStage(BaseStage):
-    """Trigger Workflow execution stage that execute another workflow object.
+    """Trigger Workflow execution stage that execute another workflow. This this
+    the core stage that allow you to create the reusable workflow object or
+    dynamic parameters workflow for common usecase.
 
     Data Validate:
         >>> stage = {
@@ -564,7 +620,11 @@ class TriggerStage(BaseStage):
         ... }
     """
 
-    trigger: str = Field(description="A trigger workflow name.")
+    trigger: str = Field(
+        description=(
+            "A trigger workflow name that should already exist on the config."
+        ),
+    )
     params: DictData = Field(
         default_factory=dict,
         description="A parameter that want to pass to workflow execution.",
@@ -572,11 +632,13 @@ class TriggerStage(BaseStage):
 
     @handler_result("Raise from TriggerStage")
     def execute(self, params: DictData) -> Result:
-        """Trigger workflow execution.
+        """Trigger another workflow execution. It will waiting the trigger
+        workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
         :rtype: Result
         """
+        # NOTE: Lazy import this workflow object.
         from . import Workflow
 
         # NOTE: Loading workflow object from trigger name.
@@ -591,7 +653,11 @@ class TriggerStage(BaseStage):
         return wf.execute(params=param2template(self.params, params))
 
 
-# NOTE: Order of parsing stage data
+# NOTE:
+#   An order of parsing stage model on the Job model with ``stages`` field.
+#   From the current build-in stages, they do not have stage that have the same
+#   fields that be cause of parsing on the Job's stages key.
+#
 Stage = Union[
     PyStage,
     BashStage,

ddeutil/workflow/utils.py

@@ -13,6 +13,7 @@ import time
 from abc import ABC, abstractmethod
 from ast import Call, Constant, Expr, Module, Name, parse
 from collections.abc import Iterator
+from dataclasses import field
 from datetime import date, datetime
 from functools import cached_property, wraps
 from hashlib import md5
@@ -32,19 +33,22 @@ except ImportError:
 from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy, str2bool
 from ddeutil.io import PathData, PathSearch, YamlFlResolve, search_env_replace
 from ddeutil.io.models.lineage import dt_now
-from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.dataclasses import dataclass
 from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, Matrix, Re
+from .conf import config
 from .exceptions import ParamValueException, UtilException
 
-logger = logging.getLogger("ddeutil.workflow")
 P = ParamSpec("P")
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 AnyModelType = type[AnyModel]
 
+logger = logging.getLogger("ddeutil.workflow")
+
 
 def get_diff_sec(dt: datetime, tz: ZoneInfo | None = None) -> int:
     """Return second value that come from diff of an input datetime and the
@@ -110,7 +114,7 @@ class ConfParams(BaseModel):
     )
 
 
-def config() -> ConfParams:
+def load_config() -> ConfParams:
     """Load Config data from ``workflows-conf.yaml`` file.
 
     Configuration Docs:
@@ -158,7 +162,7 @@ class SimLoad:
     :param externals: An external parameters
 
     Noted:
-    ---
+
         The config data should have ``type`` key for modeling validation that
     make this loader know what is config should to do pass to.
 
@@ -248,11 +252,11 @@ class Loader(SimLoad):
     ) -> DictData:
         """Override the find class method from the Simple Loader object."""
         return super().finds(
-            obj=obj, params=config(), include=include, exclude=exclude
+            obj=obj, params=load_config(), include=include, exclude=exclude
         )
 
     def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, config(), externals)
+        super().__init__(name, load_config(), externals)
 
 
 def gen_id(
@@ -275,15 +279,14 @@ def gen_id(
     if not isinstance(value, str):
         value: str = str(value)
 
-    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
     if str2bool(os.getenv("WORKFLOW_CORE_PIPELINE_ID_SIMPLE", "true")):
         return hash_str(f"{(value if sensitive else value.lower())}", n=10) + (
-            f"{datetime.now(tz=tz):%Y%m%d%H%M%S%f}" if unique else ""
+            f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else ""
         )
     return md5(
         (
             f"{(value if sensitive else value.lower())}"
-            + (f"{datetime.now(tz=tz):%Y%m%d%H%M%S%f}" if unique else "")
+            + (f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else "")
         ).encode()
     ).hexdigest()
 
@@ -317,13 +320,14 @@ class TagFunc(Protocol):
     def __call__(self, *args, **kwargs): ...
 
 
-def tag(name: str, alias: str | None = None):
+def tag(name: str, alias: str | None = None) -> Callable[P, TagFunc]:
     """Tag decorator function that set function attributes, ``tag`` and ``name``
     for making registries variable.
 
-    :param: name: A tag value for make different use-case of a function.
+    :param: name: A tag name for make different use-case of a function.
     :param: alias: A alias function name that keeping in registries. If this
         value does not supply, it will use original function name from __name__.
+    :rtype: Callable[P, TagFunc]
     """
 
     def func_internal(func: Callable[[...], Any]) -> TagFunc:
@@ -350,7 +354,7 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    for module in config().engine.registry:
+    for module in load_config().engine.registry:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(f"{module}.{submodule}")
@@ -515,29 +519,50 @@ Param = Union[
 ]
 
 
-class Result(BaseModel):
-    """Result Pydantic Model for passing parameter and receiving output from
-    the workflow execution.
+@dataclass
+class Result:
+    """Result Pydantic Model for passing and receiving data context from any
+    module execution process like stage execution, job execution, or workflow
+    execution.
+
+        For comparison property, this result will use ``status``, ``context``,
+    and ``_run_id`` fields to comparing with other result instance.
     """
 
-    status: int = Field(default=2)
-    context: DictData = Field(default_factory=dict)
+    status: int = field(default=2)
+    context: DictData = field(default_factory=dict)
+    start_at: datetime = field(default_factory=dt_now, compare=False)
+    end_at: Optional[datetime] = field(default=None, compare=False)
 
     # NOTE: Ignore this field to compare another result model with __eq__.
-    _parent_run_id: Optional[str] = PrivateAttr(default=None)
-    _run_id: Optional[str] = PrivateAttr(default=None)
+    _run_id: Optional[str] = field(default=None)
+    _parent_run_id: Optional[str] = field(default=None, compare=False)
 
     @model_validator(mode="after")
-    def __prepare_run_id(self):
-        if self._run_id is None:
-            self._run_id = gen_id("manual", unique=True)
+    def __prepare_run_id(self) -> Self:
+        """Prepare running ID which use default ID if it initialize at the first
+        time
+
+        :rtype: Self
+        """
+        self._run_id = gen_id("manual", unique=True)
         return self
 
     def set_run_id(self, running_id: str) -> Self:
+        """Set a running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
         self._run_id = running_id
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
+        """Set a parent running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
         self._parent_run_id = running_id
         return self
 
@@ -549,33 +574,55 @@ class Result(BaseModel):
     def run_id(self):
         return self._run_id
 
-    def receive(self, result: Result) -> Result:
+    def catch(self, status: int, context: DictData) -> Self:
+        """Catch the status and context to current data."""
+        self.__dict__["status"] = status
+        self.__dict__["context"].update(context)
+        return self
+
+    def receive(self, result: Result) -> Self:
+        """Receive context from another result object.
+
+        :rtype: Self
+        """
         self.__dict__["status"] = result.status
         self.__dict__["context"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
         self._parent_run_id = result.parent_run_id
         self._run_id = result.run_id
         return self
 
-    def receive_jobs(self, result: Result) -> Result:
+    def receive_jobs(self, result: Result) -> Self:
+        """Receive context from another result object that use on the workflow
+        execution which create a ``jobs`` keys on the context if it do not
+        exist.
+
+        :rtype: Self
+        """
         self.__dict__["status"] = result.status
 
         # NOTE: Check the context has jobs key.
         if "jobs" not in self.__dict__["context"]:
             self.__dict__["context"]["jobs"] = {}
-
         self.__dict__["context"]["jobs"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
         self._parent_run_id = result.parent_run_id
         self._run_id = result.run_id
         return self
 
 
-def make_exec(path: str | Path):
-    """Change mode of file to be executable file."""
+def make_exec(path: str | Path) -> None:  # pragma: no cov
+    """Change mode of file to be executable file.
+
+    :param path: A file path that want to make executable permission.
+    """
     f: Path = Path(path) if isinstance(path, str) else path
     f.chmod(f.stat().st_mode | stat.S_IEXEC)
 
 
-FILTERS: dict[str, callable] = {
+FILTERS: dict[str, callable] = {  # pragma: no cov
     "abs": abs,
     "str": str,
     "int": int,
@@ -590,17 +637,18 @@ class FilterFunc(Protocol):
 
     name: str
 
-    def __call__(self, *args, **kwargs): ...
+    def __call__(self, *args, **kwargs): ...  # pragma: no cov
 
 
-def custom_filter(name: str) -> Callable[P, TagFunc]:
+def custom_filter(name: str) -> Callable[P, FilterFunc]:
     """Custom filter decorator function that set function attributes, ``filter``
     for making filter registries variable.
 
     :param: name: A filter name for make different use-case of a function.
+    :rtype: Callable[P, FilterFunc]
     """
 
-    def func_internal(func: Callable[[...], Any]) -> TagFunc:
+    def func_internal(func: Callable[[...], Any]) -> FilterFunc:
         func.filter = name
 
         @wraps(func)
@@ -622,7 +670,7 @@ def make_filter_registry() -> dict[str, FilterRegistry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    for module in config().engine.registry_filter:
+    for module in load_config().engine.registry_filter:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -644,7 +692,10 @@ def make_filter_registry() -> dict[str, FilterRegistry]:
 def get_args_const(
     expr: str,
 ) -> tuple[str, list[Constant], dict[str, Constant]]:
-    """Get arguments and keyword-arguments from function calling string."""
+    """Get arguments and keyword-arguments from function calling string.
+
+    :rtype: tuple[str, list[Constant], dict[str, Constant]]
+    """
     try:
         mod: Module = parse(expr)
     except SyntaxError:
@@ -678,6 +729,7 @@ def get_args_const(
 
 @custom_filter("fmt")
 def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
+    """Format datetime object to string with the format."""
     if isinstance(value, datetime):
         return value.strftime(fmt)
     raise UtilException(
@@ -699,8 +751,8 @@ def map_post_filter(
     """
     for _filter in post_filter:
         func_name, _args, _kwargs = get_args_const(_filter)
-        args = [arg.value for arg in _args]
-        kwargs = {k: v.value for k, v in _kwargs.items()}
+        args: list = [arg.value for arg in _args]
+        kwargs: dict = {k: v.value for k, v in _kwargs.items()}
 
         if func_name not in filters:
             raise UtilException(
@@ -845,8 +897,12 @@ def param2template(
 
 
 def filter_func(value: Any) -> Any:
-    """Filter own created function out of any value with replace it to its
-    function name. If it is built-in function, it does not have any changing.
+    """Filter out an own created function of any value of mapping context by
+    replacing it to its function name. If it is built-in function, it does not
+    have any changing.
+
+    :param value: A value context data that want to filter out function value.
+    :type: The same type of an input ``value``.
     """
     if isinstance(value, dict):
         return {k: filter_func(value[k]) for k in value}
@@ -869,14 +925,20 @@ def dash2underscore(
     *,
     fixed: str | None = None,
 ) -> DictData:
-    """Change key name that has dash to underscore."""
+    """Change key name that has dash to underscore.
+
+    :rtype: DictData
+    """
     if key in values:
         values[(fixed or key.replace("-", "_"))] = values.pop(key)
     return values
 
 
 def cross_product(matrix: Matrix) -> Iterator[DictData]:
-    """Iterator of products value from matrix."""
+    """Iterator of products value from matrix.
+
+    :rtype: Iterator[DictData]
+    """
     yield from (
         {_k: _v for e in mapped for _k, _v in e.items()}
         for mapped in product(
@@ -897,7 +959,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
     """
     if n < 1:
         raise ValueError("n must be at least one")
-    it = iter(iterable)
+    it: Iterator[Any] = iter(iterable)
     while True:
         chunk_it = islice(it, n)
         try:
@@ -905,3 +967,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         except StopIteration:
             return
         yield chain((first_el,), chunk_it)
+
+
+def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
+    return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)