ddeutil-workflow 0.0.21__py3-none-any.whl → 0.0.23__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -70,7 +70,7 @@ logging.getLogger("schedule").setLevel(logging.INFO)
 
 __all__: TupleStr = (
     "Schedule",
-    "ScheduleWorkflow",
+    "WorkflowSchedule",
     "workflow_task_release",
     "workflow_monitor",
     "workflow_control",
@@ -78,25 +78,29 @@ __all__: TupleStr = (
 )
 
 
-class ScheduleWorkflow(BaseModel):
-    """Schedule Workflow Pydantic model that use to keep workflow model for the
-    Schedule model. it should not use Workflow model directly because on the
+class WorkflowSchedule(BaseModel):
+    """Workflow Schedule Pydantic model that use to keep workflow model for
+    the Schedule model. it should not use Workflow model directly because on the
     schedule config it can adjust crontab value that different from the Workflow
     model.
     """
 
     alias: Optional[str] = Field(
         default=None,
-        description="An alias name of workflow.",
+        description="An alias name of workflow that use for schedule model.",
     )
     name: str = Field(description="A workflow name.")
     on: list[On] = Field(
         default_factory=list,
-        description="An override On instance value.",
+        description="An override the list of On object values.",
     )
-    params: DictData = Field(
+    values: DictData = Field(
         default_factory=dict,
-        description="A parameters that want to use in workflow execution.",
+        description=(
+            "A value that want to pass to the workflow parameters when "
+            "calling release method."
+        ),
+        alias="params",
     )
 
     @model_validator(mode="before")
@@ -105,10 +109,13 @@ class ScheduleWorkflow(BaseModel):
 
         :rtype: DictData
         """
-        values["name"] = values["name"].replace(" ", "_")
+        # VALIDATE: Prepare a workflow name that should not include space.
+        if name := values.get("name"):
+            values["name"] = name.replace(" ", "_")
 
+        # VALIDATE: Add default the alias field with the name.
         if not values.get("alias"):
-            values["alias"] = values["name"]
+            values["alias"] = values.get("name")
 
         cls.__bypass_on(values)
         return values
@@ -135,6 +142,7 @@ class ScheduleWorkflow(BaseModel):
                 Loader(n, externals={}).data if isinstance(n, str) else n
                 for n in on
             ]
+
         return data
 
     @field_validator("on", mode="after")
@@ -150,19 +158,20 @@ class ScheduleWorkflow(BaseModel):
                 "The on fields should not contain duplicate on value."
             )
 
-        # WARNING:
-        # if '* * * * *' in set_ons and len(set_ons) > 1:
-        #     raise ValueError(
-        #         "If it has every minute cronjob on value, it should has only "
-        #         "one value in the on field."
-        #     )
+        if len(set_ons) > config.max_on_per_workflow:
+            raise ValueError(
+                f"The number of the on should not more than "
+                f"{config.max_on_per_workflow} crontab."
+            )
+
         return value
 
 
 class Schedule(BaseModel):
-    """Schedule Pydantic Model that use to run with scheduler package. It does
-    not equal the on value in Workflow model but it use same logic to running
-    release date with crontab interval.
+    """Schedule Pydantic model that use to run with any scheduler package.
+
+        It does not equal the on value in Workflow model but it use same logic
+    to running release date with crontab interval.
     """
 
     desc: Optional[str] = Field(
@@ -171,9 +180,9 @@ class Schedule(BaseModel):
             "A schedule description that can be string of markdown content."
         ),
     )
-    workflows: list[ScheduleWorkflow] = Field(
+    workflows: list[WorkflowSchedule] = Field(
         default_factory=list,
-        description="A list of ScheduleWorkflow models.",
+        description="A list of WorkflowSchedule models.",
     )
 
     @field_validator("desc", mode="after")
@@ -181,6 +190,7 @@ class Schedule(BaseModel):
         """Prepare description string that was created on a template.
 
         :param value: A description string value that want to dedent.
+
         :rtype: str
         """
         return dedent(value)
@@ -224,6 +234,9 @@ class Schedule(BaseModel):
         """Return the list of WorkflowTaskData object from the specific input
         datetime that mapping with the on field.
 
+            This task creation need queue to tracking release date already
+        mapped or not.
+
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
         :param externals: An external parameters that pass to the Loader object.
@@ -232,22 +245,21 @@ class Schedule(BaseModel):
         :return: Return the list of WorkflowTaskData object from the specific
             input datetime that mapping with the on field.
         """
-
-        # NOTE: Create pair of workflow and on.
         workflow_tasks: list[WorkflowTaskData] = []
         extras: DictData = externals or {}
 
         for sch_wf in self.workflows:
 
+            # NOTE: Loading workflow model from the name of workflow.
             wf: Workflow = Workflow.from_loader(sch_wf.name, externals=extras)
 
-            # NOTE: Create default list of release datetime.
+            # NOTE: Create default list of release datetime by empty list.
             if sch_wf.alias not in queue:
                 queue[sch_wf.alias]: list[datetime] = []
 
             # IMPORTANT: Create the default 'on' value if it does not passing
             #   the on field to the Schedule object.
-            ons: list[On] = wf.on.copy() if len(sch_wf.on) == 0 else sch_wf.on
+            ons: list[On] = sch_wf.on or wf.on.copy()
 
             for on in ons:
 
@@ -263,7 +275,7 @@ class Schedule(BaseModel):
                         alias=sch_wf.alias,
                         workflow=wf,
                         runner=runner,
-                        params=sch_wf.params,
+                        params=sch_wf.values,
                     ),
                 )
 

ddeutil/workflow/stage.py

@@ -51,10 +51,11 @@ from typing_extensions import Self
 from .__types import DictData, DictStr, Re, TupleStr
 from .conf import config, get_logger
 from .exceptions import StageException
+from .result import Result
 from .utils import (
     Registry,
-    Result,
     TagFunc,
+    cut_id,
     gen_id,
     make_exec,
     make_registry,
@@ -124,13 +125,16 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 run_id: str = gen_id(self.name + (self.id or ""), unique=True)
                 kwargs["run_id"] = run_id
 
+            rs_raise: Result = Result(status=1, run_id=run_id)
+
             try:
                 # NOTE: Start calling origin function with a passing args.
                 return func(self, *args, **kwargs)
             except Exception as err:
                 # NOTE: Start catching error from the stage execution.
                 logger.error(
-                    f"({run_id}) [STAGE]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
+                    f"{err}"
                 )
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
@@ -147,13 +151,12 @@ def handler_result(message: str | None = None) -> DecoratorResult:
 
                 # NOTE: Catching exception error object to result with
                 #   error_message and error keys.
-                return Result(
+                return rs_raise.catch(
                     status=1,
                     context={
                         "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                    run_id=run_id,
                 )
 
         return wrapped
@@ -339,7 +342,7 @@ class EmptyStage(BaseStage):
         :rtype: Result
         """
         logger.info(
-            f"({run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
+            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
@@ -393,7 +396,9 @@ class BashStage(BaseStage):
         f_name: str = f"{run_id}.sh"
         f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
 
-        logger.debug(f"({run_id}) [STAGE]: Start create `{f_name}` file.")
+        logger.debug(
+            f"({cut_id(run_id)}) [STAGE]: Start create `{f_name}` file."
+        )
 
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: write header of `.sh` file
@@ -425,7 +430,7 @@ class BashStage(BaseStage):
         """
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({run_id}) [STAGE]: Shell-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
             bash=bash, env=param2template(self.env, params), run_id=run_id
         ) as sh:
@@ -535,7 +540,7 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({run_id}) [STAGE]: Py-Execute: {self.name}")
+        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is vary dangerous. So, it
         #   should us the re module to validate exec-string before running.
@@ -660,7 +665,8 @@ class HookStage(BaseStage):
                 args[k] = args.pop(k.removeprefix("_"))
 
         logger.info(
-            f"({run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
+            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
+            f"{t_func.name}@{t_func.tag}"
         )
         rs: DictData = t_func(**param2template(args, params))
 
@@ -716,7 +722,9 @@ class TriggerStage(BaseStage):
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
         wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(f"({run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
+        logger.info(
+            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
+        )
         return wf.execute(
             params=param2template(self.params, params),
             run_id=run_id,

ddeutil/workflow/utils.py

@@ -9,11 +9,9 @@ import inspect
 import logging
 import stat
 import time
-from abc import ABC, abstractmethod
 from ast import Call, Constant, Expr, Module, Name, parse
 from collections.abc import Iterator
-from dataclasses import field
-from datetime import date, datetime, timedelta
+from datetime import datetime, timedelta
 from functools import wraps
 from hashlib import md5
 from importlib import import_module
@@ -21,7 +19,7 @@ from inspect import isfunction
 from itertools import chain, islice, product
 from pathlib import Path
 from random import randrange
-from typing import Any, Callable, Literal, Optional, Protocol, TypeVar, Union
+from typing import Any, Callable, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 try:
@@ -31,14 +29,11 @@ except ImportError:
 
 from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy
 from ddeutil.io import search_env_replace
-from pydantic import BaseModel, Field
-from pydantic.dataclasses import dataclass
-from pydantic.functional_validators import model_validator
-from typing_extensions import Self
+from pydantic import BaseModel
 
 from .__types import DictData, Matrix, Re
 from .conf import config
-from .exceptions import ParamValueException, UtilException
+from .exceptions import UtilException
 
 T = TypeVar("T")
 P = ParamSpec("P")
@@ -198,239 +193,7 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     return rs
 
 
-class BaseParam(BaseModel, ABC):
-    """Base Parameter that use to make Params Model."""
-
-    desc: Optional[str] = Field(
-        default=None, description="A description of parameter providing."
-    )
-    required: bool = Field(
-        default=True,
-        description="A require flag that force to pass this parameter value.",
-    )
-    type: str = Field(description="A type of parameter.")
-
-    @abstractmethod
-    def receive(self, value: Optional[Any] = None) -> Any:
-        raise NotImplementedError(
-            "Receive value and validate typing before return valid value."
-        )
-
-
-class DefaultParam(BaseParam):
-    """Default Parameter that will check default if it required. This model do
-    not implement the receive method.
-    """
-
-    required: bool = Field(
-        default=False,
-        description="A require flag for the default-able parameter value.",
-    )
-    default: Optional[str] = Field(
-        default=None,
-        description="A default value if parameter does not pass.",
-    )
-
-    @abstractmethod
-    def receive(self, value: Optional[Any] = None) -> Any:
-        raise NotImplementedError(
-            "Receive value and validate typing before return valid value."
-        )
-
-
-class DatetimeParam(DefaultParam):
-    """Datetime parameter."""
-
-    type: Literal["datetime"] = "datetime"
-    default: datetime = Field(default_factory=get_dt_now)
-
-    def receive(self, value: str | datetime | date | None = None) -> datetime:
-        """Receive value that match with datetime. If a input value pass with
-        None, it will use default value instead.
-
-        :param value: A value that want to validate with datetime parameter
-            type.
-        :rtype: datetime
-        """
-        if value is None:
-            return self.default
-
-        if isinstance(value, datetime):
-            return value
-        elif isinstance(value, date):
-            return datetime(value.year, value.month, value.day)
-        elif not isinstance(value, str):
-            raise ParamValueException(
-                f"Value that want to convert to datetime does not support for "
-                f"type: {type(value)}"
-            )
-        try:
-            return datetime.fromisoformat(value)
-        except ValueError:
-            raise ParamValueException(
-                f"Invalid isoformat string: {value!r}"
-            ) from None
-
-
-class StrParam(DefaultParam):
-    """String parameter."""
-
-    type: Literal["str"] = "str"
-
-    def receive(self, value: str | None = None) -> str | None:
-        """Receive value that match with str.
-
-        :param value: A value that want to validate with string parameter type.
-        :rtype: str | None
-        """
-        if value is None:
-            return self.default
-        return str(value)
-
-
-class IntParam(DefaultParam):
-    """Integer parameter."""
-
-    type: Literal["int"] = "int"
-    default: Optional[int] = Field(
-        default=None,
-        description="A default value if parameter does not pass.",
-    )
-
-    def receive(self, value: int | None = None) -> int | None:
-        """Receive value that match with int.
-
-        :param value: A value that want to validate with integer parameter type.
-        :rtype: int | None
-        """
-        if value is None:
-            return self.default
-        if not isinstance(value, int):
-            try:
-                return int(str(value))
-            except ValueError as err:
-                raise ParamValueException(
-                    f"Value can not convert to int, {value}, with base 10"
-                ) from err
-        return value
-
-
-class ChoiceParam(BaseParam):
-    """Choice parameter."""
-
-    type: Literal["choice"] = "choice"
-    options: list[str] = Field(description="A list of choice parameters.")
-
-    def receive(self, value: str | None = None) -> str:
-        """Receive value that match with options.
-
-        :param value: A value that want to select from the options field.
-        :rtype: str
-        """
-        # NOTE:
-        #   Return the first value in options if does not pass any input value
-        if value is None:
-            return self.options[0]
-        if value not in self.options:
-            raise ParamValueException(
-                f"{value!r} does not match any value in choice options."
-            )
-        return value
-
-
-Param = Union[
-    ChoiceParam,
-    DatetimeParam,
-    IntParam,
-    StrParam,
-]
-
-
-@dataclass
-class Result:
-    """Result Pydantic Model for passing and receiving data context from any
-    module execution process like stage execution, job execution, or workflow
-    execution.
-
-        For comparison property, this result will use ``status``, ``context``,
-    and ``_run_id`` fields to comparing with other result instance.
-    """
-
-    status: int = field(default=2)
-    context: DictData = field(default_factory=dict)
-
-    # NOTE: Ignore this field to compare another result model with __eq__.
-    run_id: Optional[str] = field(default=None)
-    parent_run_id: Optional[str] = field(default=None, compare=False)
-
-    @model_validator(mode="after")
-    def __prepare_run_id(self) -> Self:
-        """Prepare running ID which use default ID if it initialize at the first
-        time
-
-        :rtype: Self
-        """
-        self._run_id = gen_id("manual", unique=True)
-        return self
-
-    def set_run_id(self, running_id: str) -> Self:
-        """Set a running ID.
-
-        :param running_id: A running ID that want to update on this model.
-        :rtype: Self
-        """
-        self.run_id = running_id
-        return self
-
-    def set_parent_run_id(self, running_id: str) -> Self:
-        """Set a parent running ID.
-
-        :param running_id: A running ID that want to update on this model.
-        :rtype: Self
-        """
-        self.parent_run_id: str = running_id
-        return self
-
-    def catch(self, status: int, context: DictData) -> Self:
-        """Catch the status and context to current data."""
-        self.__dict__["status"] = status
-        self.__dict__["context"].update(context)
-        return self
-
-    def receive(self, result: Result) -> Self:
-        """Receive context from another result object.
-
-        :rtype: Self
-        """
-        self.__dict__["status"] = result.status
-        self.__dict__["context"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id = result.parent_run_id
-        self.run_id = result.run_id
-        return self
-
-    def receive_jobs(self, result: Result) -> Self:
-        """Receive context from another result object that use on the workflow
-        execution which create a ``jobs`` keys on the context if it do not
-        exist.
-
-        :rtype: Self
-        """
-        self.__dict__["status"] = result.status
-
-        # NOTE: Check the context has jobs key.
-        if "jobs" not in self.__dict__["context"]:
-            self.__dict__["context"]["jobs"] = {}
-        self.__dict__["context"]["jobs"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id: str = result.parent_run_id
-        self.run_id: str = result.run_id
-        return self
-
-
-def make_exec(path: str | Path) -> None:  # pragma: no cov
+def make_exec(path: str | Path) -> None:
     """Change mode of file to be executable file.
 
     :param path: A file path that want to make executable permission.
@@ -451,7 +214,9 @@ FILTERS: dict[str, callable] = {  # pragma: no cov
 
 
 class FilterFunc(Protocol):
-    """Tag Function Protocol"""
+    """Tag Function Protocol. This protocol that use to represent any callable
+    object that able to access the name attribute.
+    """
 
     name: str
 
@@ -814,3 +579,17 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
 
 def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
     return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
+
+
+def cut_id(run_id: str, *, num: int = 6):
+    """Cutting running ID with length.
+
+    Example:
+        >>> cut_id(run_id='668931127320241228100331254567')
+        '254567'
+
+    :param run_id:
+    :param num:
+    :return:
+    """
+    return run_id[-num:]