ddeutil-workflow 0.0.30__tar.gz → 0.0.32__tar.gz

{ddeutil_workflow-0.0.30/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.30
+Version: 0.0.32
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -110,7 +110,7 @@ flowchart LR
     E -.->|read| G
 ```
 
-> [!NOTE]
+> [!WARNING]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
 > with `.yml` files and all configs file from several data orchestration framework
 > tools from my experience on Data Engineer. :grimacing:
@@ -192,7 +192,17 @@ The above workflow template is main executor pipeline that you want to do. If yo
 want to schedule this workflow, you want to dynamic its parameters change base on
 execution time such as `run-date` should change base on that workflow running date.
 
-So, this package provide the `Schedule` template for this action.
+```python
+from ddeutil.workflow import Workflow, Result
+
+workflow: Workflow = Workflow.from_loader('run-py-local')
+result: Result = workflow.execute(
+   params={"source-extract": "USD-THB", "asat-dt": "2024-01-01"}
+)
+```
+
+So, this package provide the `Schedule` template for this action, and you can dynamic
+pass the parameters for changing align with that running time by the `release` prefix.
 
 ```yaml
 schedule-run-local-wf:
@@ -210,6 +220,20 @@ schedule-run-local-wf:
           asat-dt: "${{ release.logical_date }}"
 ```
 
+The main method of the `Schedule` model that use to running is `pending`. If you
+do not pass the `stop` date on this method, it will use config with `WORKFLOW_APP_STOP_BOUNDARY_DELTA`
+key for generate this stop date.
+
+```python
+from ddeutil.workflow import Schedule
+
+(
+   Schedule
+   .from_loader("schedule-run-local-wf")
+   .pending(stop=None)
+)
+```
+
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/README.md

@@ -78,7 +78,7 @@ flowchart LR
     E -.->|read| G
 ```
 
-> [!NOTE]
+> [!WARNING]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
 > with `.yml` files and all configs file from several data orchestration framework
 > tools from my experience on Data Engineer. :grimacing:
@@ -160,7 +160,17 @@ The above workflow template is main executor pipeline that you want to do. If yo
 want to schedule this workflow, you want to dynamic its parameters change base on
 execution time such as `run-date` should change base on that workflow running date.
 
-So, this package provide the `Schedule` template for this action.
+```python
+from ddeutil.workflow import Workflow, Result
+
+workflow: Workflow = Workflow.from_loader('run-py-local')
+result: Result = workflow.execute(
+   params={"source-extract": "USD-THB", "asat-dt": "2024-01-01"}
+)
+```
+
+So, this package provide the `Schedule` template for this action, and you can dynamic
+pass the parameters for changing align with that running time by the `release` prefix.
 
 ```yaml
 schedule-run-local-wf:
@@ -178,6 +188,20 @@ schedule-run-local-wf:
           asat-dt: "${{ release.logical_date }}"
 ```
 
+The main method of the `Schedule` model that use to running is `pending`. If you
+do not pass the `stop` date on this method, it will use config with `WORKFLOW_APP_STOP_BOUNDARY_DELTA`
+key for generate this stop date.
+
+```python
+from ddeutil.workflow import Schedule
+
+(
+   Schedule
+   .from_loader("schedule-run-local-wf")
+   .pending(stop=None)
+)
+```
+
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your

ddeutil_workflow-0.0.32/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.32"

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/__cron.py

@@ -845,5 +845,6 @@ __all__ = (
     "CronJob",
     "CronJobYear",
     "CronRunner",
+    "Options",
     "WEEKDAYS",
 )

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/__init__.py

@@ -47,8 +47,10 @@ from .params import (
 from .result import Result
 from .scheduler import (
     Schedule,
-    WorkflowSchedule,
+    ScheduleWorkflow,
+    schedule_control,
     schedule_runner,
+    schedule_task,
 )
 from .stage import (
     BashStage,
@@ -83,6 +85,8 @@ from .utils import (
     make_exec,
 )
 from .workflow import (
+    Release,
+    ReleaseQueue,
     Workflow,
     WorkflowTask,
 )

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/api/api.py

@@ -43,9 +43,7 @@ async def lifespan(a: FastAPI) -> AsyncIterator[State]:
     a.state.workflow_queue = {}
 
     yield {
-        "upper_queue": a.state.upper_queue,
-        "upper_result": a.state.upper_result,
-        # NOTE: Scheduler value should be contain a key of workflow workflow and
+        # NOTE: Scheduler value should be contained a key of workflow and
         #   list of datetime of queue and running.
         #
         #   ... {
@@ -88,7 +86,7 @@ if config.enable_route_workflow:
 
 # NOTE: Enable the schedule route.
 if config.enable_route_schedule:
-    from ..conf import FileLog
+    from ..conf import get_log
     from ..scheduler import schedule_task
     from .route import schedule_route
 
@@ -108,11 +106,11 @@ if config.enable_route_schedule:
                 stop=datetime.now(config.tz) + timedelta(minutes=1),
                 queue=app.state.workflow_queue,
                 threads=app.state.workflow_threads,
-                log=FileLog,
+                log=get_log(),
             )
 
     @schedule_route.on_event("startup")
-    @repeat_at(cron="*/5 * * * *")
+    @repeat_at(cron="*/5 * * * *", delay=10)
     def monitoring():
         logger.debug("[MONITOR]: Start monitoring threading.")
         snapshot_threads: list[str] = list(app.state.workflow_threads.keys())

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/api/route.py

@@ -169,7 +169,7 @@ async def get_schedules(name: str):
     )
 
 
-@schedule_route.get(path="/deploy")
+@schedule_route.get(path="/deploy/")
 async def get_deploy_schedulers(request: Request):
     snapshot = copy.deepcopy(request.state.scheduler)
     return {"schedule": snapshot}
@@ -178,9 +178,9 @@ async def get_deploy_schedulers(request: Request):
 @schedule_route.get(path="/deploy/{name}")
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
-        sch = Schedule.from_loader(name)
+        schedule = Schedule.from_loader(name)
         getter: list[dict[str, dict[str, list[datetime]]]] = []
-        for workflow in sch.workflows:
+        for workflow in schedule.workflows:
             getter.append(
                 {
                     workflow.name: {
@@ -219,7 +219,7 @@ async def add_deploy_scheduler(request: Request, name: str):
         second=0, microsecond=0
     )
 
-    # NOTE: Create pair of workflow and on from schedule model.
+    # NOTE: Create a pair of workflow and on from schedule model.
     try:
         schedule: Schedule = Schedule.from_loader(name)
     except ValueError as err:

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/conf.py

@@ -582,6 +582,10 @@ Log = Union[
 
 
 def get_log() -> type[Log]:  # pragma: no cov
+    """Get logging class that dynamic base on the config log path value.
+
+    :rtype: type[Log]
+    """
     if config.log_path.is_file():
         return SQLiteLog
     return FileLog

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/cron.py

@@ -5,16 +5,17 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Literal
+from typing import Annotated, Literal, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
+from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr, TupleStr
 from .conf import Loader
 
@@ -47,6 +48,8 @@ def interval2crontab(
         '0 0 1 * *'
         >>> interval2crontab(interval='monthly', day='tuesday', time='12:00')
         '12 0 1 * 2'
+
+    :rtype: str
     """
     d: str = "*"
     if interval == "weekly":
@@ -64,12 +67,19 @@ class On(BaseModel):
     """On Pydantic model (Warped crontab object by model).
 
     See Also:
-        * ``generate()`` is the main usecase of this schedule object.
+        * ``generate()`` is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     # NOTE: This is fields of the base schedule.
+    extras: Annotated[
+        DictData,
+        Field(
+            default_factory=dict,
+            description="An extras mapping parameters",
+        ),
+    ]
     cronjob: Annotated[
         CronJob,
         Field(description="Cron job of this schedule"),
@@ -81,13 +91,6 @@ class On(BaseModel):
             alias="timezone",
         ),
     ] = "Etc/UTC"
-    extras: Annotated[
-        DictData,
-        Field(
-            default_factory=dict,
-            description="An extras mapping parameters",
-        ),
-    ]
 
     @classmethod
     def from_value(cls, value: DictStr, externals: DictData) -> Self:
@@ -153,6 +156,7 @@ class On(BaseModel):
 
     @model_validator(mode="before")
     def __prepare_values(cls, values: DictData) -> DictData:
+        """Extract tz key from value and change name to timezone key."""
         if tz := values.pop("tz", None):
             values["timezone"] = tz
         return values
@@ -160,24 +164,55 @@ class On(BaseModel):
     @field_validator("tz")
     def __validate_tz(cls, value: str) -> str:
         """Validate timezone value that able to initialize with ZoneInfo after
-        it passing to this model in before mode."""
+        it passing to this model in before mode.
+
+        :rtype: str
+        """
         try:
             _ = ZoneInfo(value)
             return value
         except ZoneInfoNotFoundError as err:
             raise ValueError(f"Invalid timezone: {value}") from err
 
-    @field_validator("cronjob", mode="before")
-    def __prepare_cronjob(cls, value: str | CronJob) -> CronJob:
-        """Prepare crontab value that able to receive with string type."""
-        return CronJob(value) if isinstance(value, str) else value
+    @field_validator(
+        "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
+    )
+    def __prepare_cronjob(
+        cls, value: str | CronJob, info: ValidationInfo
+    ) -> CronJob:
+        """Prepare crontab value that able to receive with string type.
+        This step will get options kwargs from extras and pass to the
+        CronJob object.
+
+        :rtype: CronJob
+        """
+        extras: DictData = info.data.get("extras", {})
+        return (
+            CronJob(
+                value,
+                option={
+                    name: extras[name]
+                    for name in (f.name for f in fields(Options))
+                    if name in extras
+                },
+            )
+            if isinstance(value, str)
+            else value
+        )
 
     @field_serializer("cronjob")
     def __serialize_cronjob(self, value: CronJob) -> str:
+        """Serialize the cronjob field that store with CronJob object.
+
+        :rtype: str
+        """
         return str(value)
 
     def generate(self, start: str | datetime) -> CronRunner:
-        """Return Cron runner object."""
+        """Return Cron runner object.
+
+        :rtype: CronRunner
+        """
         if isinstance(start, str):
             start: datetime = datetime.fromisoformat(start)
         elif not isinstance(start, datetime):
@@ -187,6 +222,8 @@ class On(BaseModel):
     def next(self, start: str | datetime) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
+
+        :rtype: CronRunner
         """
         runner: CronRunner = self.generate(start=start)
 
@@ -209,7 +246,26 @@ class YearOn(On):
         Field(description="Cron job of this schedule"),
     ]
 
-    @field_validator("cronjob", mode="before")
-    def __prepare_cronjob(cls, value: str | CronJobYear) -> CronJobYear:
-        """Prepare crontab value that able to receive with string type."""
-        return CronJobYear(value) if isinstance(value, str) else value
+    @field_validator(
+        "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
+    )
+    def __prepare_cronjob(
+        cls, value: str | CronJobYear, info: ValidationInfo
+    ) -> CronJobYear:
+        """Prepare crontab value that able to receive with string type.
+
+        :rtype: CronJobYear
+        """
+        extras: DictData = info.data.get("extras", {})
+        return (
+            CronJobYear(
+                value,
+                option={
+                    name: extras[name]
+                    for name in (f.name for f in fields(Options))
+                    if name in extras
+                },
+            )
+            if isinstance(value, str)
+            else value
+        )

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/exceptions.py

@@ -29,3 +29,6 @@ class WorkflowFailException(WorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
+
+
+class ScheduleException(BaseWorkflowException): ...

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/job.py

@@ -7,6 +7,8 @@
 The job handle the lineage of stages and location of execution of stages that
 mean the job model able to define ``runs-on`` key that allow you to run this
 job.
+
+    This module include Strategy Model that use on the job strategy field.
 """
 from __future__ import annotations
 

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/params.py

@@ -3,8 +3,12 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Param Model that use for parsing incoming parameters that pass to the
+Workflow and Schedule objects.
+"""
 from __future__ import annotations
 
+import decimal
 import logging
 from abc import ABC, abstractmethod
 from datetime import date, datetime
@@ -49,7 +53,7 @@ class BaseParam(BaseModel, ABC):
 
 class DefaultParam(BaseParam):
     """Default Parameter that will check default if it required. This model do
-    not implement the receive method.
+    not implement the `receive` method.
     """
 
     required: bool = Field(
@@ -68,6 +72,15 @@ class DefaultParam(BaseParam):
         )
 
 
+# TODO: Not implement this parameter yet
+class DateParam(DefaultParam):  # pragma: no cov
+    """Date parameter."""
+
+    type: Literal["date"] = "date"
+
+    def receive(self, value: Optional[str | date] = None) -> date: ...
+
+
 class DatetimeParam(DefaultParam):
     """Datetime parameter."""
 
@@ -145,6 +158,13 @@ class IntParam(DefaultParam):
         return value
 
 
+# TODO: Not implement this parameter yet
+class DecimalParam(DefaultParam):  # pragma: no cov
+    type: Literal["decimal"] = "decimal"
+
+    def receive(self, value: float | None = None) -> decimal.Decimal: ...
+
+
 class ChoiceParam(BaseParam):
     """Choice parameter."""
 

{ddeutil_workflow-0.0.30 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/result.py

@@ -94,6 +94,7 @@ class Result:
         # NOTE: Check the context has jobs key.
         if "jobs" not in self.__dict__["context"]:
             self.__dict__["context"]["jobs"] = {}
+
         self.__dict__["context"]["jobs"].update(result.context)
 
         # NOTE: Update running ID from an incoming result.