ddeutil-workflow 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.22"
+__version__: str = "0.0.24"

ddeutil/workflow/__cron.py

@@ -736,6 +736,12 @@ class CronRunner:
         self.is_year: bool = isinstance(cron, CronJobYear)
         self.reset_flag: bool = True
 
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(CronJob('{self.cron}'), "
+            f"{self.date:%Y-%m-%d %H:%M:%S}, tz='{self.tz}')"
+        )
+
     def reset(self) -> None:
         """Resets the iterator to start time."""
         self.date: datetime = self.__start_date

ddeutil/workflow/__init__.py

@@ -3,11 +3,17 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+from .__cron import CronRunner
 from .conf import (
     Config,
     FileLog,
     Loader,
 )
+from .cron import (
+    On,
+    YearOn,
+    interval2crontab,
+)
 from .exceptions import (
     JobException,
     ParamValueException,
@@ -19,14 +25,18 @@ from .job import (
     Job,
     Strategy,
 )
-from .on import (
-    On,
-    YearOn,
-    interval2crontab,
+from .params import (
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    Param,
+    StrParam,
 )
+from .result import Result
 from .scheduler import (
     Schedule,
-    ScheduleWorkflow,
+    WorkflowSchedule,
+    schedule_runner,
 )
 from .stage import (
     BashStage,
@@ -39,16 +49,9 @@ from .stage import (
 )
 from .utils import (
     FILTERS,
-    ChoiceParam,
-    DatetimeParam,
-    DefaultParam,
     FilterFunc,
     FilterRegistry,
-    IntParam,
-    Param,
-    Result,
     ReturnTagFunc,
-    StrParam,
     TagFunc,
     batch,
     cross_product,
@@ -67,11 +70,10 @@ from .utils import (
     map_post_filter,
     not_in_template,
     param2template,
-    queue2str,
     str2template,
     tag,
 )
 from .workflow import (
     Workflow,
-    WorkflowTaskData,
+    WorkflowTask,
 )

ddeutil/workflow/api.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel
 from .__about__ import __version__
 from .conf import config, get_logger
 from .repeat import repeat_at, repeat_every
-from .scheduler import WorkflowTaskData
+from .workflow import WorkflowTask
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -34,7 +34,7 @@ class State(TypedDict):
     upper_result: dict[str, str]
     scheduler: list[str]
     workflow_threads: dict[str, Thread]
-    workflow_tasks: list[WorkflowTaskData]
+    workflow_tasks: list[WorkflowTask]
     workflow_queue: dict[str, list[datetime]]
     workflow_running: dict[str, list[datetime]]
 

ddeutil/workflow/cli.py

@@ -73,10 +73,10 @@ def schedule(
     if stop:
         stop: datetime = stop.astimezone(tz=config.tz)
 
-    from .scheduler import workflow_runner
+    from .scheduler import schedule_runner
 
     # NOTE: Start running workflow scheduler application.
-    workflow_rs: list[str] = workflow_runner(
+    workflow_rs: list[str] = schedule_runner(
         stop=stop, excluded=excluded, externals=json.loads(externals)
     )
     logger.info(f"Application run success: {workflow_rs}")

ddeutil/workflow/conf.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData
+from .__types import DictData, TupleStr
 
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 AnyModelType = type[AnyModel]
@@ -32,6 +32,19 @@ load_dotenv()
 
 env = os.getenv
 
+__all__: TupleStr = (
+    "get_logger",
+    "Config",
+    "SimLoad",
+    "Loader",
+    "get_type",
+    "config",
+    "logger",
+    "FileLog",
+    "SQLiteLog",
+    "Log",
+)
+
 
 @lru_cache
 def get_logger(name: str):
@@ -107,7 +120,10 @@ class Config:
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )
     max_on_per_workflow: int = int(
-        env("WORKFLOW_CORE_MAX_ON_PER_WORKFLOW", "5")
+        env("WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW", "5")
+    )
+    max_queue_complete_hist: int = int(
+        os.getenv("WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST", "16")
     )
 
     # NOTE: Schedule App

ddeutil/workflow/{on.py → cron.py}

@@ -61,7 +61,7 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Model (Schedule)
+    """On Pydantic model (Warped crontab object by model).
 
     See Also:
         * ``generate()`` is the main usecase of this schedule object.
@@ -197,8 +197,8 @@ class On(BaseModel):
 
 
 class YearOn(On):
-    """Implement On Year Schedule Model for limit year matrix that use by some
-    data schedule tools like AWS Glue.
+    """On with enhance Year Pydantic model for limit year matrix that use by
+    some data schedule tools like AWS Glue.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)

ddeutil/workflow/job.py

@@ -36,9 +36,9 @@ from .exceptions import (
     StageException,
     UtilException,
 )
+from .result import Result
 from .stage import Stage
 from .utils import (
-    Result,
     cross_product,
     cut_id,
     dash2underscore,
@@ -313,7 +313,7 @@ class Job(BaseModel):
         # VALIDATE: Validate stage id should not duplicate.
         rs: list[str] = []
         for stage in value:
-            name: str = stage.id or stage.name
+            name: str = stage.iden
             if name in rs:
                 raise ValueError(
                     "Stage name in jobs object should not be duplicate."

ddeutil/workflow/params.py

@@ -0,0 +1,176 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from datetime import date, datetime
+from typing import Any, Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from .__types import TupleStr
+from .exceptions import ParamValueException
+from .utils import get_dt_now
+
+logger = logging.getLogger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "ChoiceParam",
+    "DatetimeParam",
+    "IntParam",
+    "Param",
+    "StrParam",
+)
+
+
+class BaseParam(BaseModel, ABC):
+    """Base Parameter that use to make any Params Model. The type will dynamic
+    with the type field that made from literal string."""
+
+    desc: Optional[str] = Field(
+        default=None, description="A description of parameter providing."
+    )
+    required: bool = Field(
+        default=True,
+        description="A require flag that force to pass this parameter value.",
+    )
+    type: str = Field(description="A type of parameter.")
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise NotImplementedError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DefaultParam(BaseParam):
+    """Default Parameter that will check default if it required. This model do
+    not implement the receive method.
+    """
+
+    required: bool = Field(
+        default=False,
+        description="A require flag for the default-able parameter value.",
+    )
+    default: Optional[str] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise NotImplementedError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DatetimeParam(DefaultParam):
+    """Datetime parameter."""
+
+    type: Literal["datetime"] = "datetime"
+    default: datetime = Field(default_factory=get_dt_now)
+
+    def receive(self, value: str | datetime | date | None = None) -> datetime:
+        """Receive value that match with datetime. If a input value pass with
+        None, it will use default value instead.
+
+        :param value: A value that want to validate with datetime parameter
+            type.
+        :rtype: datetime
+        """
+        if value is None:
+            return self.default
+
+        if isinstance(value, datetime):
+            return value
+        elif isinstance(value, date):
+            return datetime(value.year, value.month, value.day)
+        elif not isinstance(value, str):
+            raise ParamValueException(
+                f"Value that want to convert to datetime does not support for "
+                f"type: {type(value)}"
+            )
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            raise ParamValueException(
+                f"Invalid isoformat string: {value!r}"
+            ) from None
+
+
+class StrParam(DefaultParam):
+    """String parameter."""
+
+    type: Literal["str"] = "str"
+
+    def receive(self, value: str | None = None) -> str | None:
+        """Receive value that match with str.
+
+        :param value: A value that want to validate with string parameter type.
+        :rtype: str | None
+        """
+        if value is None:
+            return self.default
+        return str(value)
+
+
+class IntParam(DefaultParam):
+    """Integer parameter."""
+
+    type: Literal["int"] = "int"
+    default: Optional[int] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
+
+    def receive(self, value: int | None = None) -> int | None:
+        """Receive value that match with int.
+
+        :param value: A value that want to validate with integer parameter type.
+        :rtype: int | None
+        """
+        if value is None:
+            return self.default
+        if not isinstance(value, int):
+            try:
+                return int(str(value))
+            except ValueError as err:
+                raise ParamValueException(
+                    f"Value can not convert to int, {value}, with base 10"
+                ) from err
+        return value
+
+
+class ChoiceParam(BaseParam):
+    """Choice parameter."""
+
+    type: Literal["choice"] = "choice"
+    options: list[str] = Field(description="A list of choice parameters.")
+
+    def receive(self, value: str | None = None) -> str:
+        """Receive value that match with options.
+
+        :param value: A value that want to select from the options field.
+        :rtype: str
+        """
+        # NOTE:
+        #   Return the first value in options if does not pass any input value
+        if value is None:
+            return self.options[0]
+        if value not in self.options:
+            raise ParamValueException(
+                f"{value!r} does not match any value in choice options."
+            )
+        return value
+
+
+Param = Union[
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    StrParam,
+]

ddeutil/workflow/result.py

@@ -0,0 +1,102 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from dataclasses import field
+from typing import Optional
+
+from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData, TupleStr
+from .utils import gen_id
+
+__all__: TupleStr = ("Result",)
+
+
+@dataclass
+class Result:
+    """Result Pydantic Model for passing and receiving data context from any
+    module execution process like stage execution, job execution, or workflow
+    execution.
+
+        For comparison property, this result will use ``status``, ``context``,
+    and ``_run_id`` fields to comparing with other result instance.
+    """
+
+    status: int = field(default=2)
+    context: DictData = field(default_factory=dict)
+    run_id: Optional[str] = field(default=None)
+
+    # NOTE: Ignore this field to compare another result model with __eq__.
+    parent_run_id: Optional[str] = field(default=None, compare=False)
+
+    @model_validator(mode="after")
+    def __prepare_run_id(self) -> Self:
+        """Prepare running ID which use default ID if it initialize at the first
+        time
+
+        :rtype: Self
+        """
+        self._run_id = gen_id("manual", unique=True)
+        return self
+
+    def set_run_id(self, running_id: str) -> Self:
+        """Set a running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
+        self.run_id = running_id
+        return self
+
+    def set_parent_run_id(self, running_id: str) -> Self:
+        """Set a parent running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
+        self.parent_run_id: str = running_id
+        return self
+
+    def catch(self, status: int, context: DictData) -> Self:
+        """Catch the status and context to current data."""
+        self.__dict__["status"] = status
+        self.__dict__["context"].update(context)
+        return self
+
+    def receive(self, result: Result) -> Self:
+        """Receive context from another result object.
+
+        :rtype: Self
+        """
+        self.__dict__["status"] = result.status
+        self.__dict__["context"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
+        self.parent_run_id = result.parent_run_id
+        self.run_id = result.run_id
+        return self
+
+    def receive_jobs(self, result: Result) -> Self:
+        """Receive context from another result object that use on the workflow
+        execution which create a ``jobs`` keys on the context if it do not
+        exist.
+
+        :rtype: Self
+        """
+        self.__dict__["status"] = result.status
+
+        # NOTE: Check the context has jobs key.
+        if "jobs" not in self.__dict__["context"]:
+            self.__dict__["context"]["jobs"] = {}
+        self.__dict__["context"]["jobs"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
+        self.parent_run_id: str = result.parent_run_id
+        self.run_id: str = result.run_id
+        return self

ddeutil/workflow/route.py

@@ -17,8 +17,8 @@ from pydantic import BaseModel
 from . import Workflow
 from .__types import DictData
 from .conf import Loader, config, get_logger
+from .result import Result
 from .scheduler import Schedule
-from .utils import Result
 
 logger = get_logger("ddeutil.workflow")
 workflow = APIRouter(