ddeutil-workflow 0.0.22__tar.gz → 0.0.23__tar.gz

{ddeutil_workflow-0.0.22/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.22
+Version: 0.0.23
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.23/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.23"

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/__init__.py

@@ -24,9 +24,17 @@ from .on import (
     YearOn,
     interval2crontab,
 )
+from .params import (
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    Param,
+    StrParam,
+)
+from .result import Result
 from .scheduler import (
     Schedule,
-    ScheduleWorkflow,
+    WorkflowSchedule,
 )
 from .stage import (
     BashStage,
@@ -39,16 +47,9 @@ from .stage import (
 )
 from .utils import (
     FILTERS,
-    ChoiceParam,
-    DatetimeParam,
-    DefaultParam,
     FilterFunc,
     FilterRegistry,
-    IntParam,
-    Param,
-    Result,
     ReturnTagFunc,
-    StrParam,
     TagFunc,
     batch,
     cross_product,

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/api.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel
 from .__about__ import __version__
 from .conf import config, get_logger
 from .repeat import repeat_at, repeat_every
-from .scheduler import WorkflowTaskData
+from .workflow import WorkflowTaskData
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/job.py

@@ -36,9 +36,9 @@ from .exceptions import (
     StageException,
     UtilException,
 )
+from .result import Result
 from .stage import Stage
 from .utils import (
-    Result,
     cross_product,
     cut_id,
     dash2underscore,
@@ -313,7 +313,7 @@ class Job(BaseModel):
         # VALIDATE: Validate stage id should not duplicate.
         rs: list[str] = []
         for stage in value:
-            name: str = stage.id or stage.name
+            name: str = stage.iden
             if name in rs:
                 raise ValueError(
                     "Stage name in jobs object should not be duplicate."

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/on.py

@@ -61,7 +61,7 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Model (Schedule)
+    """On Pydantic model (Warped crontab object by model).
 
     See Also:
         * ``generate()`` is the main usecase of this schedule object.
@@ -197,8 +197,8 @@ class On(BaseModel):
 
 
 class YearOn(On):
-    """Implement On Year Schedule Model for limit year matrix that use by some
-    data schedule tools like AWS Glue.
+    """On with enhance Year Pydantic model for limit year matrix that use by
+    some data schedule tools like AWS Glue.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)

ddeutil_workflow-0.0.23/src/ddeutil/workflow/params.py

@@ -0,0 +1,176 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from datetime import date, datetime
+from typing import Any, Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from .__types import TupleStr
+from .exceptions import ParamValueException
+from .utils import get_dt_now
+
+logger = logging.getLogger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "ChoiceParam",
+    "DatetimeParam",
+    "IntParam",
+    "Param",
+    "StrParam",
+)
+
+
+class BaseParam(BaseModel, ABC):
+    """Base Parameter that use to make any Params Model. The type will dynamic
+    with the type field that made from literal string."""
+
+    desc: Optional[str] = Field(
+        default=None, description="A description of parameter providing."
+    )
+    required: bool = Field(
+        default=True,
+        description="A require flag that force to pass this parameter value.",
+    )
+    type: str = Field(description="A type of parameter.")
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise NotImplementedError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DefaultParam(BaseParam):
+    """Default Parameter that will check default if it required. This model do
+    not implement the receive method.
+    """
+
+    required: bool = Field(
+        default=False,
+        description="A require flag for the default-able parameter value.",
+    )
+    default: Optional[str] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise NotImplementedError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DatetimeParam(DefaultParam):
+    """Datetime parameter."""
+
+    type: Literal["datetime"] = "datetime"
+    default: datetime = Field(default_factory=get_dt_now)
+
+    def receive(self, value: str | datetime | date | None = None) -> datetime:
+        """Receive value that match with datetime. If a input value pass with
+        None, it will use default value instead.
+
+        :param value: A value that want to validate with datetime parameter
+            type.
+        :rtype: datetime
+        """
+        if value is None:
+            return self.default
+
+        if isinstance(value, datetime):
+            return value
+        elif isinstance(value, date):
+            return datetime(value.year, value.month, value.day)
+        elif not isinstance(value, str):
+            raise ParamValueException(
+                f"Value that want to convert to datetime does not support for "
+                f"type: {type(value)}"
+            )
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            raise ParamValueException(
+                f"Invalid isoformat string: {value!r}"
+            ) from None
+
+
+class StrParam(DefaultParam):
+    """String parameter."""
+
+    type: Literal["str"] = "str"
+
+    def receive(self, value: str | None = None) -> str | None:
+        """Receive value that match with str.
+
+        :param value: A value that want to validate with string parameter type.
+        :rtype: str | None
+        """
+        if value is None:
+            return self.default
+        return str(value)
+
+
+class IntParam(DefaultParam):
+    """Integer parameter."""
+
+    type: Literal["int"] = "int"
+    default: Optional[int] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
+
+    def receive(self, value: int | None = None) -> int | None:
+        """Receive value that match with int.
+
+        :param value: A value that want to validate with integer parameter type.
+        :rtype: int | None
+        """
+        if value is None:
+            return self.default
+        if not isinstance(value, int):
+            try:
+                return int(str(value))
+            except ValueError as err:
+                raise ParamValueException(
+                    f"Value can not convert to int, {value}, with base 10"
+                ) from err
+        return value
+
+
+class ChoiceParam(BaseParam):
+    """Choice parameter."""
+
+    type: Literal["choice"] = "choice"
+    options: list[str] = Field(description="A list of choice parameters.")
+
+    def receive(self, value: str | None = None) -> str:
+        """Receive value that match with options.
+
+        :param value: A value that want to select from the options field.
+        :rtype: str
+        """
+        # NOTE:
+        #   Return the first value in options if does not pass any input value
+        if value is None:
+            return self.options[0]
+        if value not in self.options:
+            raise ParamValueException(
+                f"{value!r} does not match any value in choice options."
+            )
+        return value
+
+
+Param = Union[
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    StrParam,
+]

ddeutil_workflow-0.0.23/src/ddeutil/workflow/result.py

@@ -0,0 +1,102 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from dataclasses import field
+from typing import Optional
+
+from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData, TupleStr
+from .utils import gen_id
+
+__all__: TupleStr = ("Result",)
+
+
+@dataclass
+class Result:
+    """Result Pydantic Model for passing and receiving data context from any
+    module execution process like stage execution, job execution, or workflow
+    execution.
+
+        For comparison property, this result will use ``status``, ``context``,
+    and ``_run_id`` fields to comparing with other result instance.
+    """
+
+    status: int = field(default=2)
+    context: DictData = field(default_factory=dict)
+    run_id: Optional[str] = field(default=None)
+
+    # NOTE: Ignore this field to compare another result model with __eq__.
+    parent_run_id: Optional[str] = field(default=None, compare=False)
+
+    @model_validator(mode="after")
+    def __prepare_run_id(self) -> Self:
+        """Prepare running ID which use default ID if it initialize at the first
+        time
+
+        :rtype: Self
+        """
+        self._run_id = gen_id("manual", unique=True)
+        return self
+
+    def set_run_id(self, running_id: str) -> Self:
+        """Set a running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
+        self.run_id = running_id
+        return self
+
+    def set_parent_run_id(self, running_id: str) -> Self:
+        """Set a parent running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
+        self.parent_run_id: str = running_id
+        return self
+
+    def catch(self, status: int, context: DictData) -> Self:
+        """Catch the status and context to current data."""
+        self.__dict__["status"] = status
+        self.__dict__["context"].update(context)
+        return self
+
+    def receive(self, result: Result) -> Self:
+        """Receive context from another result object.
+
+        :rtype: Self
+        """
+        self.__dict__["status"] = result.status
+        self.__dict__["context"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
+        self.parent_run_id = result.parent_run_id
+        self.run_id = result.run_id
+        return self
+
+    def receive_jobs(self, result: Result) -> Self:
+        """Receive context from another result object that use on the workflow
+        execution which create a ``jobs`` keys on the context if it do not
+        exist.
+
+        :rtype: Self
+        """
+        self.__dict__["status"] = result.status
+
+        # NOTE: Check the context has jobs key.
+        if "jobs" not in self.__dict__["context"]:
+            self.__dict__["context"]["jobs"] = {}
+        self.__dict__["context"]["jobs"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
+        self.parent_run_id: str = result.parent_run_id
+        self.run_id: str = result.run_id
+        return self

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/route.py

@@ -17,8 +17,8 @@ from pydantic import BaseModel
 from . import Workflow
 from .__types import DictData
 from .conf import Loader, config, get_logger
+from .result import Result
 from .scheduler import Schedule
-from .utils import Result
 
 logger = get_logger("ddeutil.workflow")
 workflow = APIRouter(

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/scheduler.py

@@ -70,7 +70,7 @@ logging.getLogger("schedule").setLevel(logging.INFO)
 
 __all__: TupleStr = (
     "Schedule",
-    "ScheduleWorkflow",
+    "WorkflowSchedule",
     "workflow_task_release",
     "workflow_monitor",
     "workflow_control",
@@ -78,25 +78,29 @@ __all__: TupleStr = (
 )
 
 
-class ScheduleWorkflow(BaseModel):
-    """Schedule Workflow Pydantic model that use to keep workflow model for the
-    Schedule model. it should not use Workflow model directly because on the
+class WorkflowSchedule(BaseModel):
+    """Workflow Schedule Pydantic model that use to keep workflow model for
+    the Schedule model. it should not use Workflow model directly because on the
     schedule config it can adjust crontab value that different from the Workflow
     model.
     """
 
     alias: Optional[str] = Field(
         default=None,
-        description="An alias name of workflow.",
+        description="An alias name of workflow that use for schedule model.",
     )
     name: str = Field(description="A workflow name.")
     on: list[On] = Field(
         default_factory=list,
-        description="An override On instance value.",
+        description="An override the list of On object values.",
     )
-    params: DictData = Field(
+    values: DictData = Field(
         default_factory=dict,
-        description="A parameters that want to use in workflow execution.",
+        description=(
+            "A value that want to pass to the workflow parameters when "
+            "calling release method."
+        ),
+        alias="params",
     )
 
     @model_validator(mode="before")
@@ -105,10 +109,13 @@ class ScheduleWorkflow(BaseModel):
 
         :rtype: DictData
         """
-        values["name"] = values["name"].replace(" ", "_")
+        # VALIDATE: Prepare a workflow name that should not include space.
+        if name := values.get("name"):
+            values["name"] = name.replace(" ", "_")
 
+        # VALIDATE: Add default the alias field with the name.
         if not values.get("alias"):
-            values["alias"] = values["name"]
+            values["alias"] = values.get("name")
 
         cls.__bypass_on(values)
         return values
@@ -135,6 +142,7 @@ class ScheduleWorkflow(BaseModel):
                 Loader(n, externals={}).data if isinstance(n, str) else n
                 for n in on
             ]
+
         return data
 
     @field_validator("on", mode="after")
@@ -150,19 +158,20 @@ class ScheduleWorkflow(BaseModel):
                 "The on fields should not contain duplicate on value."
             )
 
-        # WARNING:
-        # if '* * * * *' in set_ons and len(set_ons) > 1:
-        #     raise ValueError(
-        #         "If it has every minute cronjob on value, it should has only "
-        #         "one value in the on field."
-        #     )
+        if len(set_ons) > config.max_on_per_workflow:
+            raise ValueError(
+                f"The number of the on should not more than "
+                f"{config.max_on_per_workflow} crontab."
+            )
+
         return value
 
 
 class Schedule(BaseModel):
-    """Schedule Pydantic Model that use to run with scheduler package. It does
-    not equal the on value in Workflow model but it use same logic to running
-    release date with crontab interval.
+    """Schedule Pydantic model that use to run with any scheduler package.
+
+        It does not equal the on value in Workflow model but it use same logic
+    to running release date with crontab interval.
     """
 
     desc: Optional[str] = Field(
@@ -171,9 +180,9 @@ class Schedule(BaseModel):
             "A schedule description that can be string of markdown content."
         ),
     )
-    workflows: list[ScheduleWorkflow] = Field(
+    workflows: list[WorkflowSchedule] = Field(
         default_factory=list,
-        description="A list of ScheduleWorkflow models.",
+        description="A list of WorkflowSchedule models.",
     )
 
     @field_validator("desc", mode="after")
@@ -181,6 +190,7 @@ class Schedule(BaseModel):
         """Prepare description string that was created on a template.
 
         :param value: A description string value that want to dedent.
+
         :rtype: str
         """
         return dedent(value)
@@ -224,6 +234,9 @@ class Schedule(BaseModel):
         """Return the list of WorkflowTaskData object from the specific input
         datetime that mapping with the on field.
 
+            This task creation need queue to tracking release date already
+        mapped or not.
+
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
         :param externals: An external parameters that pass to the Loader object.
@@ -232,22 +245,21 @@ class Schedule(BaseModel):
         :return: Return the list of WorkflowTaskData object from the specific
             input datetime that mapping with the on field.
         """
-
-        # NOTE: Create pair of workflow and on.
         workflow_tasks: list[WorkflowTaskData] = []
         extras: DictData = externals or {}
 
         for sch_wf in self.workflows:
 
+            # NOTE: Loading workflow model from the name of workflow.
             wf: Workflow = Workflow.from_loader(sch_wf.name, externals=extras)
 
-            # NOTE: Create default list of release datetime.
+            # NOTE: Create default list of release datetime by empty list.
             if sch_wf.alias not in queue:
                 queue[sch_wf.alias]: list[datetime] = []
 
             # IMPORTANT: Create the default 'on' value if it does not passing
             #   the on field to the Schedule object.
-            ons: list[On] = wf.on.copy() if len(sch_wf.on) == 0 else sch_wf.on
+            ons: list[On] = sch_wf.on or wf.on.copy()
 
             for on in ons:
 
@@ -263,7 +275,7 @@ class Schedule(BaseModel):
                         alias=sch_wf.alias,
                         workflow=wf,
                         runner=runner,
-                        params=sch_wf.params,
+                        params=sch_wf.values,
                     ),
                 )
 

{ddeutil_workflow-0.0.22 → ddeutil_workflow-0.0.23}/src/ddeutil/workflow/stage.py

@@ -51,9 +51,9 @@ from typing_extensions import Self
 from .__types import DictData, DictStr, Re, TupleStr
 from .conf import config, get_logger
 from .exceptions import StageException
+from .result import Result
 from .utils import (
     Registry,
-    Result,
     TagFunc,
     cut_id,
     gen_id,