ddeutil-workflow 0.0.38__py3-none-any.whl → 0.0.40__py3-none-any.whl

ddeutil/workflow/job.py

@@ -40,12 +40,8 @@ from .exceptions import (
 )
 from .result import Result, Status
 from .stages import Stage
-from .templates import has_template
-from .utils import (
-    cross_product,
-    filter_func,
-    gen_id,
-)
+from .templates import has_template, param2template
+from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -54,6 +50,7 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
+    "TriggerState",
     "RunsOn",
     "RunsOnLocal",
     "RunsOnSelfHosted",
@@ -208,6 +205,16 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
+class TriggerState(str, Enum):
+    waiting: str = "waiting"
+    passed: str = "passed"
+    skipped: str = "skipped"
+    failed: str = "failed"
+
+    def is_waiting(self):
+        return self.value == "waiting"
+
+
 class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
@@ -312,13 +319,21 @@ class Job(BaseModel):
         description="A target node for this job to use for execution.",
         alias="runs-on",
     )
+    condition: Optional[str] = Field(
+        default=None,
+        description="A job condition statement to allow job executable.",
+        alias="if",
+    )
     stages: list[Stage] = Field(
         default_factory=list,
         description="A list of Stage of this job.",
     )
     trigger_rule: TriggerRules = Field(
         default=TriggerRules.all_success,
-        description="A trigger rule of tracking needed jobs.",
+        description=(
+            "A trigger rule of tracking needed jobs if feature will use when "
+            "the `raise_error` did not set from job and stage executions."
+        ),
         alias="trigger-rule",
     )
     needs: list[str] = Field(
@@ -382,12 +397,87 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
-    def check_needs(self, jobs: dict[str, Any]) -> bool:
+    def check_needs(
+        self, jobs: dict[str, Any]
+    ) -> TriggerState:  # pragma: no cov
         """Return True if job's need exists in an input list of job's ID.
 
+        :param jobs: A mapping of job model and its ID.
+
+        :rtype: TriggerState
+        """
+        if not self.needs:
+            return TriggerState.passed
+
+        def make_return(result: bool) -> TriggerState:
+            return TriggerState.passed if result else TriggerState.failed
+
+        need_exist: dict[str, Any] = {
+            need: jobs[need] for need in self.needs if need in jobs
+        }
+        if len(need_exist) != len(self.needs):
+            return TriggerState.waiting
+        elif all("skipped" in need_exist[job] for job in need_exist):
+            return TriggerState.skipped
+        elif self.trigger_rule == TriggerRules.all_done:
+            return TriggerState.passed
+        elif self.trigger_rule == TriggerRules.all_success:
+            rs = all(
+                k not in need_exist[job]
+                for k in ("errors", "skipped")
+                for job in need_exist
+            )
+        elif self.trigger_rule == TriggerRules.all_failed:
+            rs = all("errors" in need_exist[job] for job in need_exist)
+        elif self.trigger_rule == TriggerRules.one_success:
+            rs = sum(
+                k not in need_exist[job]
+                for k in ("errors", "skipped")
+                for job in need_exist
+            ) + 1 == len(self.needs)
+        elif self.trigger_rule == TriggerRules.one_failed:
+            rs = sum("errors" in need_exist[job] for job in need_exist) == 1
+        elif self.trigger_rule == TriggerRules.none_skipped:
+            rs = all("skipped" not in need_exist[job] for job in need_exist)
+        elif self.trigger_rule == TriggerRules.none_failed:
+            rs = all("errors" not in need_exist[job] for job in need_exist)
+        else:  # pragma: no cov
+            raise NotImplementedError(
+                f"Trigger rule: {self.trigger_rule} does not support yet."
+            )
+        return make_return(rs)
+
+    def is_skipped(self, params: DictData | None = None) -> bool:
+        """Return true if condition of this job do not correct. This process
+        use build-in eval function to execute the if-condition.
+
+        :raise JobException: When it has any error raise from the eval
+            condition statement.
+        :raise JobException: When return type of the eval condition statement
+            does not return with boolean type.
+
+        :param params: (DictData) A parameters that want to pass to condition
+            template.
+
         :rtype: bool
         """
-        return all(need in jobs for need in self.needs)
+        if self.condition is None:
+            return False
+
+        params: DictData = {} if params is None else params
+
+        try:
+            # WARNING: The eval build-in function is very dangerous. So, it
+            #   should use the `re` module to validate eval-string before
+            #   running.
+            rs: bool = eval(
+                param2template(self.condition, params), globals() | params, {}
+            )
+            if not isinstance(rs, bool):
+                raise TypeError("Return type of condition does not be boolean")
+            return not rs
+        except Exception as err:
+            raise JobException(f"{err.__class__.__name__}: {err}") from err
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
@@ -436,7 +526,9 @@ class Job(BaseModel):
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
 
-        if self.strategy.is_set():
+        if "SKIP" in output:  # pragma: no cov
+            to["jobs"][_id] = output["SKIP"]
+        elif self.strategy.is_set():
             to["jobs"][_id] = {"strategies": output, **errors}
         else:
             _output = output.get(next(iter(output), "FIRST"), {})
@@ -458,8 +550,8 @@ class Job(BaseModel):
         multithread on this metrics to the `stages` field of this job.
 
         :param params: An input parameters that use on job execution.
-        :param run_id: A job running ID for this execution.
-        :param parent_run_id: A parent workflow running ID for this release.
+        :param run_id: (str) A job running ID.
+        :param parent_run_id: (str) A parent workflow running ID.
         :param result: (Result) A result object for keeping context and status
             data.
         :param event: (Event) An event manager that pass to the
@@ -559,6 +651,7 @@ def local_execute_strategy(
 
         if stage.is_skipped(params=context):
             result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
+            stage.set_outputs(output={"skipped": True}, to=context)
             continue
 
         if event and event.is_set():
@@ -623,9 +716,6 @@ def local_execute_strategy(
                 },
             )
 
-        # NOTE: Remove the current stage object for saving memory.
-        del stage
-
     return result.catch(
         status=Status.SUCCESS,
         context={
@@ -680,7 +770,17 @@ def local_execute(
 
         for strategy in job.strategy.make():
 
-            # TODO: stop and raise error if the event was set.
+            if event and event.is_set():  # pragma: no cov
+                return result.catch(
+                    status=Status.FAILED,
+                    context={
+                        "errors": JobException(
+                            "Job strategy was canceled from event that had set "
+                            "before strategy execution."
+                        ).to_dict()
+                    },
+                )
+
             local_execute_strategy(
                 job=job,
                 strategy=strategy,
@@ -694,12 +794,22 @@ def local_execute(
 
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
-
     result.trace.info(
         f"[JOB]: Start multithreading: {job.strategy.max_parallel} threads "
         f"with {ls} mode."
     )
 
+    if event and event.is_set():  # pragma: no cov
+        return result.catch(
+            status=Status.FAILED,
+            context={
+                "errors": JobException(
+                    "Job strategy was canceled from event that had set "
+                    "before strategy execution."
+                ).to_dict()
+            },
+        )
+
     # IMPORTANT: Start running strategy execution by multithreading because
     #   it will run by strategy values without waiting previous execution.
     with ThreadPoolExecutor(

ddeutil/workflow/scheduler.py

@@ -31,6 +31,7 @@ from concurrent.futures import (
 from datetime import datetime, timedelta
 from functools import wraps
 from heapq import heappop, heappush
+from pathlib import Path
 from textwrap import dedent
 from threading import Thread
 from typing import Callable, Optional, TypedDict, Union
@@ -52,7 +53,7 @@ except ImportError:  # pragma: no cov
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
 from .audit import Audit, get_audit
-from .conf import Loader, config, get_logger
+from .conf import Loader, SimLoad, config, get_logger
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
 from .result import Result, Status
@@ -266,6 +267,8 @@ class Schedule(BaseModel):
         :param externals: An external parameters that want to pass to Loader
             object.
 
+        :raise ValueError: If the type does not match with current object.
+
         :rtype: Self
         """
         loader: Loader = Loader(name, externals=(externals or {}))
@@ -281,6 +284,42 @@ class Schedule(BaseModel):
 
         return cls.model_validate(obj=loader_data)
 
+    @classmethod
+    def from_path(
+        cls,
+        name: str,
+        path: Path,
+        externals: DictData | None = None,
+    ) -> Self:
+        """Create Schedule instance from the SimLoad object that receive an
+        input schedule name and conf path. The loader object will use this
+        schedule name to searching configuration data of this schedule model
+        in conf path.
+
+        :param name: (str) A schedule name that want to pass to Loader object.
+        :param path: (Path) A config path that want to search.
+        :param externals: An external parameters that want to pass to Loader
+            object.
+
+        :raise ValueError: If the type does not match with current object.
+
+        :rtype: Self
+        """
+        loader: SimLoad = SimLoad(
+            name, conf_path=path, externals=(externals or {})
+        )
+
+        # NOTE: Validate the config type match with current connection model
+        if loader.type != cls.__name__:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+
+        loader_data: DictData = copy.deepcopy(loader.data)
+
+        # NOTE: Add name to loader data
+        loader_data["name"] = name.replace(" ", "_")
+
+        return cls.model_validate(obj=loader_data)
+
     def tasks(
         self,
         start_date: datetime,

ddeutil/workflow/stages.py

@@ -41,7 +41,7 @@ from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
-from typing import Optional, Union
+from typing import Annotated, Optional, Union
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
@@ -230,7 +230,10 @@ class BaseStage(BaseModel, ABC):
 
             ... (iii) to: {
                         'stages': {
-                            '<stage-id>': {'outputs': {'foo': 'bar'}}
+                            '<stage-id>': {
+                                'outputs': {'foo': 'bar'},
+                                'skipped': False
+                            }
                         }
                     }
 
@@ -255,8 +258,12 @@ class BaseStage(BaseModel, ABC):
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
-
-        to["stages"][_id] = {"outputs": output, **errors}
+        skipping: dict[str, bool] = (
+            {"skipped": output.pop("skipped", False)}
+            if "skipped" in output
+            else {}
+        )
+        to["stages"][_id] = {"outputs": output, **skipping, **errors}
         return to
 
     def is_skipped(self, params: DictData | None = None) -> bool:
@@ -539,19 +546,11 @@ class PyStage(BaseStage):
 
         :rtype: DictData
         """
-        # NOTE: The output will fileter unnecessary keys from locals.
-        lc: DictData = output.get("locals", {})
+        lc: DictData = output.pop("locals", {})
+        gb: DictData = output.pop("globals", {})
         super().set_outputs(
-            (
-                {k: lc[k] for k in self.filter_locals(lc)}
-                | ({"errors": output["errors"]} if "errors" in output else {})
-            ),
-            to=to,
+            {k: lc[k] for k in self.filter_locals(lc)} | output, to=to
         )
-
-        # NOTE: Override value that changing from the globals that pass via the
-        #   exec function.
-        gb: DictData = output.get("globals", {})
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
@@ -572,17 +571,13 @@ class PyStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        # NOTE: Replace the run statement that has templating value.
-        run: str = param2template(dedent(self.run), params)
-
-        # NOTE: create custom globals value that will pass to exec function.
-        _globals: DictData = (
+        lc: DictData = {}
+        gb: DictData = (
             globals()
             | params
             | param2template(self.vars, params)
             | {"result": result}
         )
-        lc: DictData = {}
 
         # NOTE: Start exec the run statement.
         result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
@@ -591,14 +586,12 @@ class PyStage(BaseStage):
             "check your statement be safe before execute."
         )
 
-        # TODO: Add Python systax wrapper for checking dangerous code before run
-        #   this statement.
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
-        exec(run, _globals, lc)
+        exec(param2template(dedent(self.run), params), gb, lc)
 
         return result.catch(
-            status=Status.SUCCESS, context={"locals": lc, "globals": _globals}
+            status=Status.SUCCESS, context={"locals": lc, "globals": gb}
         )
 
 
@@ -795,7 +788,9 @@ class ParallelStage(BaseStage):  # pragma: no cov
         ... }
     """
 
-    parallel: dict[str, list[Stage]] = Field()
+    parallel: dict[str, list[Stage]] = Field(
+        description="A mapping of parallel branch ID.",
+    )
     max_parallel_core: int = Field(default=2)
 
     @staticmethod
@@ -807,9 +802,10 @@ class ParallelStage(BaseStage):  # pragma: no cov
     ) -> DictData:
         """Task execution method for passing a branch to each thread.
 
-        :param branch:
-        :param params:
-        :param result:
+        :param branch: A branch ID.
+        :param params: A parameter data that want to use in this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :param stages:
 
         :rtype: DictData
@@ -1008,7 +1004,7 @@ class IfStage(BaseStage):  # pragma: no cov
 
     """
 
-    case: str
+    case: str = Field(description="A case condition for routing.")
     match: list[dict[str, Union[str, Stage]]]
 
     def execute(
@@ -1016,6 +1012,18 @@ class IfStage(BaseStage):  # pragma: no cov
     ) -> Result: ...
 
 
+class RaiseStage(BaseStage):  # pragma: no cov
+    message: str = Field(
+        description="An error message that want to raise",
+        alias="raise",
+    )
+
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
+        raise StageException(self.message)
+
+
 # TODO: Not implement this stages yet
 class HookStage(BaseStage):  # pragma: no cov
     hook: str
@@ -1050,6 +1058,11 @@ class VirtualPyStage(PyStage):  # pragma: no cov
 
     def create_py_file(self, py: str, run_id: str | None): ...
 
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
+        return super().execute(params, result=result)
+
 
 # TODO: Not implement this stages yet
 class SensorStage(BaseStage):  # pragma: no cov
@@ -1064,12 +1077,16 @@ class SensorStage(BaseStage):  # pragma: no cov
 #   From the current build-in stages, they do not have stage that have the same
 #   fields that because of parsing on the Job's stages key.
 #
-Stage = Union[
-    EmptyStage,
-    BashStage,
-    CallStage,
-    TriggerStage,
-    ForEachStage,
-    ParallelStage,
-    PyStage,
+Stage = Annotated[
+    Union[
+        EmptyStage,
+        BashStage,
+        CallStage,
+        TriggerStage,
+        ForEachStage,
+        ParallelStage,
+        PyStage,
+        RaiseStage,
+    ],
+    Field(union_mode="smart"),
 ]

ddeutil/workflow/templates.py

@@ -18,7 +18,7 @@ try:
 except ImportError:
     from typing_extensions import ParamSpec
 
-from ddeutil.core import getdot, hasdot, import_string
+from ddeutil.core import getdot, import_string
 from ddeutil.io import search_env_replace
 
 from .__types import DictData, Re
@@ -59,7 +59,8 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
     """Custom filter decorator function that set function attributes, ``filter``
     for making filter registries variable.
 
-    :param: name: A filter name for make different use-case of a function.
+    :param: name: (str) A filter name for make different use-case of a function.
+
     :rtype: Callable[P, FilterFunc]
     """
 
@@ -108,7 +109,7 @@ def get_args_const(
 ) -> tuple[str, list[Constant], dict[str, Constant]]:
     """Get arguments and keyword-arguments from function calling string.
 
-    :param expr: An expr string value.
+    :param expr: (str) An expr string value.
 
     :rtype: tuple[str, list[Constant], dict[str, Constant]]
     """
@@ -154,7 +155,7 @@ def get_args_from_filter(
     and validate it with the filter functions mapping dict.
 
     :param ft:
-    :param filters:
+    :param filters: A mapping of filter registry.
 
     :rtype: tuple[str, FilterRegistry, list[Any], dict[Any, Any]]
     """
@@ -185,7 +186,7 @@ def map_post_filter(
 
     :param value: A string value that want to map with filter function.
     :param post_filter: A list of post-filter function name.
-    :param filters: A filter registry.
+    :param filters: A mapping of filter registry.
 
     :rtype: T
     """
@@ -203,8 +204,8 @@ def map_post_filter(
         except Exception as err:
             logger.warning(str(err))
             raise UtilException(
-                f"The post-filter function: {func_name} does not fit with "
-                f"{value} (type: {type(value).__name__})."
+                f"The post-filter: {func_name!r} does not fit with {value!r} "
+                f"(type: {type(value).__name__})."
             ) from None
     return value
 
@@ -258,10 +259,10 @@ def str2template(
     with the workflow parameter types that is `str`, `int`, `datetime`, and
     `list`.
 
-    :param value: A string value that want to map with params
-    :param params: A parameter value that getting with matched regular
-        expression.
-    :param filters:
+    :param value: (str) A string value that want to map with params.
+    :param params: (DictData) A parameter value that getting with matched
+        regular expression.
+    :param filters: A mapping of filter registry.
 
     :rtype: str
     """
@@ -281,11 +282,14 @@ def str2template(
             for i in (found.post_filters.strip().removeprefix("|").split("|"))
             if i != ""
         ]
-        if not hasdot(caller, params):
-            raise UtilException(f"The params does not set caller: {caller!r}.")
 
         # NOTE: from validate step, it guarantees that caller exists in params.
-        getter: Any = getdot(caller, params)
+        try:
+            getter: Any = getdot(caller, params)
+        except ValueError as err:
+            raise UtilException(
+                f"Params does not set caller: {caller!r}."
+            ) from err
 
         # NOTE:
         #   If type of getter caller is not string type, and it does not use to
@@ -301,25 +305,33 @@ def str2template(
 
         value: str = value.replace(found.full, getter, 1)
 
+    if value == "None":
+        return None
+
     return search_env_replace(value)
 
 
-def param2template(value: T, params: DictData) -> T:
+def param2template(
+    value: T,
+    params: DictData,
+    filters: dict[str, FilterRegistry] | None = None,
+) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
     :param value: A value that want to map with params
     :param params: A parameter value that getting with matched regular
         expression.
+    :param filters: A filter mapping for mapping with `map_post_filter` func.
 
     :rtype: T
     :returns: An any getter value from the params input.
     """
-    filters: dict[str, FilterRegistry] = make_filter_registry()
+    filters: dict[str, FilterRegistry] = filters or make_filter_registry()
     if isinstance(value, dict):
-        return {k: param2template(value[k], params) for k in value}
+        return {k: param2template(value[k], params, filters) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([param2template(i, params) for i in value])
+        return type(value)([param2template(i, params, filters) for i in value])
     elif not isinstance(value, str):
         return value
     return str2template(value, params, filters=filters)
@@ -329,8 +341,9 @@ def param2template(value: T, params: DictData) -> T:
 def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
     """Format datetime object to string with the format.
 
-    :param value: A datetime value that want to format to string value.
-    :param fmt: A format string pattern that passing to the `dt.strftime`
+    :param value: (datetime) A datetime value that want to format to string
+        value.
+    :param fmt: (str) A format string pattern that passing to the `dt.strftime`
         method.
 
     :rtype: str
@@ -340,3 +353,9 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
     raise UtilException(
         "This custom function should pass input value with datetime type."
     )
+
+
+@custom_filter("coalesce")  # pragma: no cov
+def coalesce(value: T | None, default: Any) -> T:
+    """Coalesce with default value if the main value is None."""
+    return default if value is None else value