ddeutil-workflow 0.0.21__py3-none-any.whl → 0.0.23__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.21"
+__version__: str = "0.0.23"

ddeutil/workflow/__init__.py

@@ -24,9 +24,17 @@ from .on import (
     YearOn,
     interval2crontab,
 )
+from .params import (
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    Param,
+    StrParam,
+)
+from .result import Result
 from .scheduler import (
     Schedule,
-    ScheduleWorkflow,
+    WorkflowSchedule,
 )
 from .stage import (
     BashStage,
@@ -39,16 +47,9 @@ from .stage import (
 )
 from .utils import (
     FILTERS,
-    ChoiceParam,
-    DatetimeParam,
-    DefaultParam,
     FilterFunc,
     FilterRegistry,
-    IntParam,
-    Param,
-    Result,
     ReturnTagFunc,
-    StrParam,
     TagFunc,
     batch,
     cross_product,

ddeutil/workflow/api.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel
 from .__about__ import __version__
 from .conf import config, get_logger
 from .repeat import repeat_at, repeat_every
-from .scheduler import WorkflowTaskData
+from .workflow import WorkflowTaskData
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")

ddeutil/workflow/conf.py

@@ -106,6 +106,9 @@ class Config:
     max_poking_pool_worker: int = int(
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )
+    max_on_per_workflow: int = int(
+        env("WORKFLOW_CORE_MAX_ON_PER_WORKFLOW", "5")
+    )
 
     # NOTE: Schedule App
     max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
@@ -462,6 +465,7 @@ class FileLog(BaseLog):
 
         :param excluded: An excluded list of key name that want to pass in the
             model_dump method.
+
         :rtype: Self
         """
         # NOTE: Check environ variable was set for real writing.

ddeutil/workflow/job.py

@@ -22,7 +22,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
@@ -36,10 +36,11 @@ from .exceptions import (
     StageException,
     UtilException,
 )
+from .result import Result
 from .stage import Stage
 from .utils import (
-    Result,
     cross_product,
+    cut_id,
     dash2underscore,
     filter_func,
     gen_id,
@@ -312,7 +313,7 @@ class Job(BaseModel):
         # VALIDATE: Validate stage id should not duplicate.
         rs: list[str] = []
         for stage in value:
-            name: str = stage.id or stage.name
+            name: str = stage.iden
             if name in rs:
                 raise ValueError(
                     "Stage name in jobs object should not be duplicate."
@@ -346,6 +347,13 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
+    def check_needs(self, jobs: dict[str, Any]) -> bool:
+        """Return True if job's need exists in an input list of job's ID.
+
+        :rtype: bool
+        """
+        return all(need in jobs for need in self.needs)
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the receive context. The
         result from execution will pass to value of ``strategies`` key.
@@ -427,6 +435,7 @@ class Job(BaseModel):
         """
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
+        rs: Result = Result(run_id=run_id)
 
         # PARAGRAPH:
         #
@@ -447,14 +456,18 @@ class Job(BaseModel):
         for stage in self.stages:
 
             if stage.is_skipped(params=context):
-                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
+                logger.info(
+                    f"({cut_id(run_id)}) [JOB]: Skip stage: {stage.iden!r}"
+                )
                 continue
 
-            logger.info(f"({run_id}) [JOB]: Execute stage: {stage.iden!r}")
+            logger.info(
+                f"({cut_id(run_id)}) [JOB]: Execute stage: {stage.iden!r}"
+            )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({run_id}) [JOB]: ... Matrix: {strategy}")
+                logger.info(f"({cut_id(run_id)}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -463,7 +476,7 @@ class Job(BaseModel):
                     "Job strategy was canceled from event that had set before "
                     "strategy execution."
                 )
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -478,7 +491,6 @@ class Job(BaseModel):
                             "error_message": error_msg,
                         },
                     },
-                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -506,14 +518,14 @@ class Job(BaseModel):
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({cut_id(run_id)}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                return Result(
+                return rs.catch(
                     status=1,
                     context={
                         strategy_id: {
@@ -523,13 +535,12 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
-                    run_id=run_id,
                 )
 
             # NOTE: Remove the current stage object for saving memory.
             del stage
 
-        return Result(
+        return rs.catch(
             status=0,
             context={
                 strategy_id: {
@@ -537,7 +548,6 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
-            run_id=run_id,
         )
 
     def execute(self, params: DictData, run_id: str | None = None) -> Result:
@@ -619,7 +629,7 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
@@ -631,7 +641,7 @@ class Job(BaseModel):
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({cut_id(run_id)}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -649,7 +659,7 @@ class Job(BaseModel):
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({run_id}) [JOB]: Fail-fast catching:\n\t"
+                    f"({cut_id(run_id)}) [JOB]: Fail-fast catching:\n\t"
                     f"{future.exception()}"
                 )
                 context.update(
@@ -680,7 +690,7 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-        rs_final: Result = Result()
+        rs_final: Result = Result(run_id=run_id)
         context: DictData = {}
         status: int = 0
 
@@ -690,7 +700,7 @@ class Job(BaseModel):
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({run_id}) [JOB]: All-completed catching:\n\t"
+                    f"({cut_id(run_id)}) [JOB]: All-completed catching:\n\t"
                     f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(

ddeutil/workflow/on.py

@@ -61,7 +61,7 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Model (Schedule)
+    """On Pydantic model (Warped crontab object by model).
 
     See Also:
         * ``generate()`` is the main usecase of this schedule object.
@@ -189,13 +189,16 @@ class On(BaseModel):
         date that given from input.
         """
         runner: CronRunner = self.generate(start=start)
+
+        # NOTE: ship the next date of runner object that create from start.
         _ = runner.next
+
         return runner
 
 
 class YearOn(On):
-    """Implement On Year Schedule Model for limit year matrix that use by some
-    data schedule tools like AWS Glue.
+    """On with enhance Year Pydantic model for limit year matrix that use by
+    some data schedule tools like AWS Glue.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)

ddeutil/workflow/params.py

@@ -0,0 +1,176 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from datetime import date, datetime
+from typing import Any, Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from .__types import TupleStr
+from .exceptions import ParamValueException
+from .utils import get_dt_now
+
+logger = logging.getLogger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "ChoiceParam",
+    "DatetimeParam",
+    "IntParam",
+    "Param",
+    "StrParam",
+)
+
+
+class BaseParam(BaseModel, ABC):
+    """Base Parameter that use to make any Params Model. The type will dynamic
+    with the type field that made from literal string."""
+
+    desc: Optional[str] = Field(
+        default=None, description="A description of parameter providing."
+    )
+    required: bool = Field(
+        default=True,
+        description="A require flag that force to pass this parameter value.",
+    )
+    type: str = Field(description="A type of parameter.")
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise NotImplementedError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DefaultParam(BaseParam):
+    """Default Parameter that will check default if it required. This model do
+    not implement the receive method.
+    """
+
+    required: bool = Field(
+        default=False,
+        description="A require flag for the default-able parameter value.",
+    )
+    default: Optional[str] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise NotImplementedError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DatetimeParam(DefaultParam):
+    """Datetime parameter."""
+
+    type: Literal["datetime"] = "datetime"
+    default: datetime = Field(default_factory=get_dt_now)
+
+    def receive(self, value: str | datetime | date | None = None) -> datetime:
+        """Receive value that match with datetime. If a input value pass with
+        None, it will use default value instead.
+
+        :param value: A value that want to validate with datetime parameter
+            type.
+        :rtype: datetime
+        """
+        if value is None:
+            return self.default
+
+        if isinstance(value, datetime):
+            return value
+        elif isinstance(value, date):
+            return datetime(value.year, value.month, value.day)
+        elif not isinstance(value, str):
+            raise ParamValueException(
+                f"Value that want to convert to datetime does not support for "
+                f"type: {type(value)}"
+            )
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            raise ParamValueException(
+                f"Invalid isoformat string: {value!r}"
+            ) from None
+
+
+class StrParam(DefaultParam):
+    """String parameter."""
+
+    type: Literal["str"] = "str"
+
+    def receive(self, value: str | None = None) -> str | None:
+        """Receive value that match with str.
+
+        :param value: A value that want to validate with string parameter type.
+        :rtype: str | None
+        """
+        if value is None:
+            return self.default
+        return str(value)
+
+
+class IntParam(DefaultParam):
+    """Integer parameter."""
+
+    type: Literal["int"] = "int"
+    default: Optional[int] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
+
+    def receive(self, value: int | None = None) -> int | None:
+        """Receive value that match with int.
+
+        :param value: A value that want to validate with integer parameter type.
+        :rtype: int | None
+        """
+        if value is None:
+            return self.default
+        if not isinstance(value, int):
+            try:
+                return int(str(value))
+            except ValueError as err:
+                raise ParamValueException(
+                    f"Value can not convert to int, {value}, with base 10"
+                ) from err
+        return value
+
+
+class ChoiceParam(BaseParam):
+    """Choice parameter."""
+
+    type: Literal["choice"] = "choice"
+    options: list[str] = Field(description="A list of choice parameters.")
+
+    def receive(self, value: str | None = None) -> str:
+        """Receive value that match with options.
+
+        :param value: A value that want to select from the options field.
+        :rtype: str
+        """
+        # NOTE:
+        #   Return the first value in options if does not pass any input value
+        if value is None:
+            return self.options[0]
+        if value not in self.options:
+            raise ParamValueException(
+                f"{value!r} does not match any value in choice options."
+            )
+        return value
+
+
+Param = Union[
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    StrParam,
+]

ddeutil/workflow/result.py

@@ -0,0 +1,102 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from dataclasses import field
+from typing import Optional
+
+from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData, TupleStr
+from .utils import gen_id
+
+__all__: TupleStr = ("Result",)
+
+
+@dataclass
+class Result:
+    """Result Pydantic Model for passing and receiving data context from any
+    module execution process like stage execution, job execution, or workflow
+    execution.
+
+        For comparison property, this result will use ``status``, ``context``,
+    and ``_run_id`` fields to comparing with other result instance.
+    """
+
+    status: int = field(default=2)
+    context: DictData = field(default_factory=dict)
+    run_id: Optional[str] = field(default=None)
+
+    # NOTE: Ignore this field to compare another result model with __eq__.
+    parent_run_id: Optional[str] = field(default=None, compare=False)
+
+    @model_validator(mode="after")
+    def __prepare_run_id(self) -> Self:
+        """Prepare running ID which use default ID if it initialize at the first
+        time
+
+        :rtype: Self
+        """
+        self._run_id = gen_id("manual", unique=True)
+        return self
+
+    def set_run_id(self, running_id: str) -> Self:
+        """Set a running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
+        self.run_id = running_id
+        return self
+
+    def set_parent_run_id(self, running_id: str) -> Self:
+        """Set a parent running ID.
+
+        :param running_id: A running ID that want to update on this model.
+        :rtype: Self
+        """
+        self.parent_run_id: str = running_id
+        return self
+
+    def catch(self, status: int, context: DictData) -> Self:
+        """Catch the status and context to current data."""
+        self.__dict__["status"] = status
+        self.__dict__["context"].update(context)
+        return self
+
+    def receive(self, result: Result) -> Self:
+        """Receive context from another result object.
+
+        :rtype: Self
+        """
+        self.__dict__["status"] = result.status
+        self.__dict__["context"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
+        self.parent_run_id = result.parent_run_id
+        self.run_id = result.run_id
+        return self
+
+    def receive_jobs(self, result: Result) -> Self:
+        """Receive context from another result object that use on the workflow
+        execution which create a ``jobs`` keys on the context if it do not
+        exist.
+
+        :rtype: Self
+        """
+        self.__dict__["status"] = result.status
+
+        # NOTE: Check the context has jobs key.
+        if "jobs" not in self.__dict__["context"]:
+            self.__dict__["context"]["jobs"] = {}
+        self.__dict__["context"]["jobs"].update(result.context)
+
+        # NOTE: Update running ID from an incoming result.
+        self.parent_run_id: str = result.parent_run_id
+        self.run_id: str = result.run_id
+        return self

ddeutil/workflow/route.py

@@ -17,8 +17,8 @@ from pydantic import BaseModel
 from . import Workflow
 from .__types import DictData
 from .conf import Loader, config, get_logger
+from .result import Result
 from .scheduler import Schedule
-from .utils import Result
 
 logger = get_logger("ddeutil.workflow")
 workflow = APIRouter(