ddeutil-workflow 0.0.30__py3-none-any.whl → 0.0.31__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.30"
+__version__: str = "0.0.31"

ddeutil/workflow/__cron.py

@@ -845,5 +845,6 @@ __all__ = (
     "CronJob",
     "CronJobYear",
     "CronRunner",
+    "Options",
     "WEEKDAYS",
 )

ddeutil/workflow/__init__.py

@@ -47,8 +47,10 @@ from .params import (
 from .result import Result
 from .scheduler import (
     Schedule,
-    WorkflowSchedule,
+    ScheduleWorkflow,
+    schedule_control,
     schedule_runner,
+    schedule_task,
 )
 from .stage import (
     BashStage,
@@ -83,6 +85,8 @@ from .utils import (
     make_exec,
 )
 from .workflow import (
+    Release,
+    ReleaseQueue,
     Workflow,
     WorkflowTask,
 )

ddeutil/workflow/conf.py

@@ -582,6 +582,10 @@ Log = Union[
 
 
 def get_log() -> type[Log]:  # pragma: no cov
+    """Get logging class that dynamic base on the config log path value.
+
+    :rtype: type[Log]
+    """
     if config.log_path.is_file():
         return SQLiteLog
     return FileLog

ddeutil/workflow/cron.py

@@ -5,16 +5,17 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Literal
+from typing import Annotated, Literal, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
+from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr, TupleStr
 from .conf import Loader
 
@@ -47,6 +48,8 @@ def interval2crontab(
         '0 0 1 * *'
         >>> interval2crontab(interval='monthly', day='tuesday', time='12:00')
         '12 0 1 * 2'
+
+    :rtype: str
     """
     d: str = "*"
     if interval == "weekly":
@@ -64,12 +67,19 @@ class On(BaseModel):
     """On Pydantic model (Warped crontab object by model).
 
     See Also:
-        * ``generate()`` is the main usecase of this schedule object.
+        * ``generate()`` is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     # NOTE: This is fields of the base schedule.
+    extras: Annotated[
+        DictData,
+        Field(
+            default_factory=dict,
+            description="An extras mapping parameters",
+        ),
+    ]
     cronjob: Annotated[
         CronJob,
         Field(description="Cron job of this schedule"),
@@ -81,13 +91,6 @@ class On(BaseModel):
             alias="timezone",
         ),
     ] = "Etc/UTC"
-    extras: Annotated[
-        DictData,
-        Field(
-            default_factory=dict,
-            description="An extras mapping parameters",
-        ),
-    ]
 
     @classmethod
     def from_value(cls, value: DictStr, externals: DictData) -> Self:
@@ -153,6 +156,7 @@ class On(BaseModel):
 
     @model_validator(mode="before")
     def __prepare_values(cls, values: DictData) -> DictData:
+        """Extract tz key from value and change name to timezone key."""
         if tz := values.pop("tz", None):
             values["timezone"] = tz
         return values
@@ -160,24 +164,55 @@ class On(BaseModel):
     @field_validator("tz")
     def __validate_tz(cls, value: str) -> str:
         """Validate timezone value that able to initialize with ZoneInfo after
-        it passing to this model in before mode."""
+        it passing to this model in before mode.
+
+        :rtype: str
+        """
         try:
             _ = ZoneInfo(value)
             return value
         except ZoneInfoNotFoundError as err:
             raise ValueError(f"Invalid timezone: {value}") from err
 
-    @field_validator("cronjob", mode="before")
-    def __prepare_cronjob(cls, value: str | CronJob) -> CronJob:
-        """Prepare crontab value that able to receive with string type."""
-        return CronJob(value) if isinstance(value, str) else value
+    @field_validator(
+        "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
+    )
+    def __prepare_cronjob(
+        cls, value: str | CronJob, info: ValidationInfo
+    ) -> CronJob:
+        """Prepare crontab value that able to receive with string type.
+        This step will get options kwargs from extras and pass to the
+        CronJob object.
+
+        :rtype: CronJob
+        """
+        extras: DictData = info.data.get("extras", {})
+        return (
+            CronJob(
+                value,
+                option={
+                    name: extras[name]
+                    for name in (f.name for f in fields(Options))
+                    if name in extras
+                },
+            )
+            if isinstance(value, str)
+            else value
+        )
 
     @field_serializer("cronjob")
     def __serialize_cronjob(self, value: CronJob) -> str:
+        """Serialize the cronjob field that store with CronJob object.
+
+        :rtype: str
+        """
         return str(value)
 
     def generate(self, start: str | datetime) -> CronRunner:
-        """Return Cron runner object."""
+        """Return Cron runner object.
+
+        :rtype: CronRunner
+        """
         if isinstance(start, str):
             start: datetime = datetime.fromisoformat(start)
         elif not isinstance(start, datetime):
@@ -187,6 +222,8 @@ class On(BaseModel):
     def next(self, start: str | datetime) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
+
+        :rtype: CronRunner
         """
         runner: CronRunner = self.generate(start=start)
 
@@ -209,7 +246,26 @@ class YearOn(On):
         Field(description="Cron job of this schedule"),
     ]
 
-    @field_validator("cronjob", mode="before")
-    def __prepare_cronjob(cls, value: str | CronJobYear) -> CronJobYear:
-        """Prepare crontab value that able to receive with string type."""
-        return CronJobYear(value) if isinstance(value, str) else value
+    @field_validator(
+        "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
+    )
+    def __prepare_cronjob(
+        cls, value: str | CronJobYear, info: ValidationInfo
+    ) -> CronJobYear:
+        """Prepare crontab value that able to receive with string type.
+
+        :rtype: CronJobYear
+        """
+        extras: DictData = info.data.get("extras", {})
+        return (
+            CronJobYear(
+                value,
+                option={
+                    name: extras[name]
+                    for name in (f.name for f in fields(Options))
+                    if name in extras
+                },
+            )
+            if isinstance(value, str)
+            else value
+        )

ddeutil/workflow/exceptions.py

@@ -29,3 +29,6 @@ class WorkflowFailException(WorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
+
+
+class ScheduleException(BaseWorkflowException): ...

ddeutil/workflow/params.py

@@ -5,6 +5,7 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import decimal
 import logging
 from abc import ABC, abstractmethod
 from datetime import date, datetime
@@ -49,7 +50,7 @@ class BaseParam(BaseModel, ABC):
 
 class DefaultParam(BaseParam):
     """Default Parameter that will check default if it required. This model do
-    not implement the receive method.
+    not implement the `receive` method.
     """
 
     required: bool = Field(
@@ -68,6 +69,15 @@ class DefaultParam(BaseParam):
         )
 
 
+# TODO: Not implement this parameter yet
+class DateParam(DefaultParam):
+    """Date parameter."""
+
+    type: Literal["date"] = "date"
+
+    def receive(self, value: Optional[str | date] = None) -> date: ...
+
+
 class DatetimeParam(DefaultParam):
     """Datetime parameter."""
 
@@ -145,6 +155,13 @@ class IntParam(DefaultParam):
         return value
 
 
+# TODO: Not implement this parameter yet
+class DecimalParam(DefaultParam):
+    type: Literal["decimal"] = "decimal"
+
+    def receive(self, value: float | None = None) -> decimal.Decimal: ...
+
+
 class ChoiceParam(BaseParam):
     """Choice parameter."""
 

ddeutil/workflow/result.py

@@ -94,6 +94,7 @@ class Result:
         # NOTE: Check the context has jobs key.
         if "jobs" not in self.__dict__["context"]:
             self.__dict__["context"]["jobs"] = {}
+
         self.__dict__["context"]["jobs"].update(result.context)
 
         # NOTE: Update running ID from an incoming result.

ddeutil/workflow/scheduler.py

@@ -33,7 +33,7 @@ from functools import wraps
 from heapq import heappop, heappush
 from textwrap import dedent
 from threading import Thread
-from typing import Callable, Optional, TypedDict
+from typing import Callable, Optional, TypedDict, Union
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import field_validator, model_validator
@@ -41,7 +41,7 @@ from typing_extensions import Self
 
 try:
     from typing import ParamSpec
-except ImportError:
+except ImportError:  # pragma: no cov
     from typing_extensions import ParamSpec
 
 try:
@@ -53,11 +53,9 @@ from .__cron import CronRunner
 from .__types import DictData, TupleStr
 from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
-from .exceptions import WorkflowException
-from .utils import (
-    batch,
-    delay,
-)
+from .exceptions import ScheduleException, WorkflowException
+from .result import Result
+from .utils import batch, delay
 from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
 
 P = ParamSpec("P")
@@ -69,7 +67,7 @@ logging.getLogger("schedule").setLevel(logging.INFO)
 
 __all__: TupleStr = (
     "Schedule",
-    "WorkflowSchedule",
+    "ScheduleWorkflow",
     "schedule_task",
     "monitor",
     "schedule_control",
@@ -79,8 +77,8 @@ __all__: TupleStr = (
 )
 
 
-class WorkflowSchedule(BaseModel):
-    """Workflow Schedule Pydantic model that use to keep workflow model for
+class ScheduleWorkflow(BaseModel):
+    """Schedule Workflow Pydantic model that use to keep workflow model for
     the Schedule model. it should not use Workflow model directly because on the
     schedule config it can adjust crontab value that different from the Workflow
     model.
@@ -233,9 +231,9 @@ class Schedule(BaseModel):
             "A schedule description that can be string of markdown content."
         ),
     )
-    workflows: list[WorkflowSchedule] = Field(
+    workflows: list[ScheduleWorkflow] = Field(
         default_factory=list,
-        description="A list of WorkflowSchedule models.",
+        description="A list of ScheduleWorkflow models.",
     )
 
     @field_validator("desc", mode="after")
@@ -258,7 +256,7 @@ class Schedule(BaseModel):
         an input schedule name. The loader object will use this schedule name to
         searching configuration data of this schedule model in conf path.
 
-        :param name: A schedule name that want to pass to Loader object.
+        :param name: (str) A schedule name that want to pass to Loader object.
         :param externals: An external parameters that want to pass to Loader
             object.
 
@@ -277,6 +275,37 @@ class Schedule(BaseModel):
 
         return cls.model_validate(obj=loader_data)
 
+    @classmethod
+    def extract_tasks(
+        cls,
+        schedules: list[str],
+        start_date: datetime,
+        queue: dict[str, ReleaseQueue],
+        externals: DictData | None = None,
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from all schedule object that
+        include in an input schedules argument.
+
+        :param schedules: A list of schedule name that will use `from_loader`
+            method.
+        :param start_date: A start date that get from the workflow schedule.
+        :param queue: A mapping of name and list of datetime for queue.
+        :param externals: An external parameters that pass to the Loader object.
+
+        :rtype: list[WorkflowTask]
+        """
+        tasks: list[WorkflowTask] = []
+        for name in schedules:
+            schedule: Schedule = Schedule.from_loader(name, externals=externals)
+            tasks.extend(
+                schedule.tasks(
+                    start_date,
+                    queue=queue,
+                    externals=externals,
+                ),
+            )
+        return tasks
+
     def tasks(
         self,
         start_date: datetime,
@@ -311,7 +340,8 @@ class Schedule(BaseModel):
         return workflow_tasks
 
 
-ReturnCancelJob = Callable[P, Optional[CancelJob]]
+ResultOrCancelJob = Union[type[CancelJob], Result]
+ReturnCancelJob = Callable[P, ResultOrCancelJob]
 DecoratorCancelJob = Callable[[ReturnCancelJob], ReturnCancelJob]
 
 
@@ -326,24 +356,25 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
     """
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:  # pragma: no cov
-        try:
 
-            @wraps(func)
-            def wrapper(*args, **kwargs):
+        @wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> ResultOrCancelJob:
+            try:
                 return func(*args, **kwargs)
+            except Exception as err:
+                logger.exception(err)
+                if cancel_on_failure:
+                    return CancelJob
+                raise err
 
-            return wrapper
-
-        except Exception as err:
-            logger.exception(err)
-            if cancel_on_failure:
-                return CancelJob
-            raise err
+        return wrapper
 
     return decorator
 
 
 class ReleaseThread(TypedDict):
+    """TypeDict for the release thread."""
+
     thread: Thread
     start_date: datetime
 
@@ -358,11 +389,13 @@ def schedule_task(
     queue: dict[str, ReleaseQueue],
     threads: ReleaseThreads,
     log: type[Log],
-) -> CancelJob | None:
-    """Workflow task generator that create release pair of workflow and on to
-    the threading in background.
+) -> type[CancelJob] | None:
+    """Schedule task function that generate thread of workflow task release
+    method in background. This function do the same logic as the workflow poke
+    method, but it runs with map of schedules and the on values.
 
-        This workflow task will start every minute at ':02' second.
+        This schedule task start runs every minute at ':02' second and it does
+    not allow you to run with offset time.
 
     :param tasks: A list of WorkflowTask object.
     :param stop: A stop datetime object that force stop running scheduler.
@@ -370,7 +403,7 @@ def schedule_task(
     :param threads: A mapping of alias name and Thread object.
     :param log: A log class that want to make log object.
 
-    :rtype: CancelJob | None
+    :rtype: type[CancelJob] | None
     """
     current_date: datetime = datetime.now(tz=config.tz)
     if current_date > stop.replace(tzinfo=config.tz):
@@ -410,12 +443,17 @@ def schedule_task(
         current_release: datetime = current_date.replace(
             second=0, microsecond=0
         )
-        if (first_date := q.first_queue.date) != current_release:
+        if (first_date := q.first_queue.date) > current_release:
             logger.debug(
                 f"[WORKFLOW]: Skip schedule "
                 f"{first_date:%Y-%m-%d %H:%M:%S} for : {task.alias!r}"
             )
             continue
+        elif first_date < current_release:  # pragma: no cov
+            raise ScheduleException(
+                "The first release date from queue should not less than current"
+                "release date."
+            )
 
         # NOTE: Pop the latest release and push it to running.
         release: Release = heappop(q.queue)
@@ -445,7 +483,7 @@ def schedule_task(
 
         delay()
 
-    logger.debug(f"[SCHEDULE]: End schedule release {'=' * 80}")
+    logger.debug(f"[SCHEDULE]: End schedule task {'=' * 80}")
 
 
 def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
@@ -455,9 +493,7 @@ def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
     :param threads: A mapping of Thread object and its name.
     :type threads: ReleaseThreads
     """
-    logger.debug(
-        "[MONITOR]: Start checking long running workflow release task."
-    )
+    logger.debug("[MONITOR]: Start checking long running schedule task.")
 
     snapshot_threads: list[str] = list(threads.keys())
     for t_name in snapshot_threads:
@@ -476,12 +512,15 @@ def schedule_control(
     *,
     log: type[Log] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Scheduler control function that running every minute.
+    """Scheduler control function that run the chuck of schedules every minute
+    and this function release monitoring thread for tracking undead thread in
+    the background.
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: A datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
-    :param log:
+    :param log: A log class that use on the workflow task release for writing
+        its release log context.
 
     :rtype: list[str]
     """
@@ -493,8 +532,11 @@ def schedule_control(
             "Should install schedule package before use this module."
         ) from None
 
+    # NOTE: Get default logging.
     log: type[Log] = log or get_log()
     scheduler: Scheduler = Scheduler()
+
+    # NOTE: Create the start and stop datetime.
     start_date: datetime = datetime.now(tz=config.tz)
     stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
 
@@ -506,25 +548,15 @@ def schedule_control(
         second=0, microsecond=0
     ) + timedelta(minutes=1)
 
-    # NOTE: Start create workflow tasks from list of schedule name.
-    tasks: list[WorkflowTask] = []
-    for name in schedules:
-        schedule: Schedule = Schedule.from_loader(name, externals=externals)
-        tasks.extend(
-            schedule.tasks(
-                start_date_waiting,
-                queue=queue,
-                externals=externals,
-            ),
-        )
-
     # NOTE: This schedule job will start every minute at :02 seconds.
     (
         scheduler.every(1)
         .minutes.at(":02")
         .do(
             schedule_task,
-            tasks=tasks,
+            tasks=Schedule.extract_tasks(
+                schedules, start_date_waiting, queue, externals=externals
+            ),
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -533,7 +565,8 @@ def schedule_control(
         .tag("control")
     )
 
-    # NOTE: Checking zombie task with schedule job will start every 5 minute.
+    # NOTE: Checking zombie task with schedule job will start every 5 minute at
+    #   :10 seconds.
     (
         scheduler.every(5)
         .minutes.at(":10")
@@ -579,16 +612,15 @@ def schedule_runner(
     externals: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Schedule runner function for start submit the ``schedule_control`` func
-    in multiprocessing pool with chunk of schedule config that exists in config
-    path by ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS``.
+    """Schedule runner function it the multiprocess controller function for
+    split the setting schedule to the `schedule_control` function on the
+    process pool. It chunks schedule configs that exists in config
+    path by `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` value.
 
     :param stop: A stop datetime object that force stop running scheduler.
     :param externals:
     :param excluded: A list of schedule name that want to exclude from finding.
 
-    :rtype: list[str]
-
         This function will get all workflows that include on value that was
     created in config path and chuck it with application config variable
     ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS`` env var to multiprocess executor
@@ -600,7 +632,9 @@ def schedule_runner(
                                         --> thread of release task 01 02
                             ==> schedule --> thread of release task 02 01
                                         --> thread of release task 02 02
-            ==> process 02
+            ==> process 02  ==> ...
+
+    :rtype: list[str]
     """
     results: list[str] = []
 

ddeutil/workflow/stage.py

@@ -328,7 +328,7 @@ class BashStage(BaseStage):
     If your current OS is Windows, it will run on the bash in the WSL.
 
         I get some limitation when I run shell statement with the built-in
-    supprocess package. It does not good enough to use multiline statement.
+    subprocess package. It does not good enough to use multiline statement.
     Thus, I add writing ``.sh`` file before execution process for fix this
     issue.
 
@@ -665,3 +665,15 @@ Stage = Union[
     TriggerStage,
     EmptyStage,
 ]
+
+
+# TODO: Not implement this stages yet
+class ParallelStage(BaseModel):
+    parallel: list[Stage]
+    max_parallel_core: int = Field(default=2)
+
+
+# TODO: Not implement this stages yet
+class ForEachStage(BaseModel):
+    foreach: list[str]
+    stages: list[Stage]

ddeutil/workflow/utils.py

@@ -21,10 +21,9 @@ from zoneinfo import ZoneInfo
 from ddeutil.core import hash_str
 
 from .__types import DictData, Matrix
-from .conf import config
 
 T = TypeVar("T")
-
+UTC = ZoneInfo("UTC")
 logger = logging.getLogger("ddeutil.workflow")
 
 
@@ -37,7 +36,7 @@ def get_dt_now(
     :param offset:
     :return: The current datetime object that use an input timezone or UTC.
     """
-    return datetime.now(tz=(tz or ZoneInfo("UTC"))) - timedelta(seconds=offset)
+    return datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)
 
 
 def get_diff_sec(
@@ -52,17 +51,42 @@ def get_diff_sec(
     """
     return round(
         (
-            dt
-            - datetime.now(tz=(tz or ZoneInfo("UTC")))
-            - timedelta(seconds=offset)
+            dt - datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)
         ).total_seconds()
     )
 
 
-def wait_a_minute(now: datetime, second: float = 2) -> None:  # pragma: no cov
+def reach_next_minute(
+    dt: datetime, tz: ZoneInfo | None = None, offset: float = 0.0
+) -> bool:
+    """Check this datetime object is not in range of minute level on the current
+    datetime.
+    """
+    diff: float = (
+        dt.replace(second=0, microsecond=0)
+        - (
+            get_dt_now(tz=(tz or UTC), offset=offset).replace(
+                second=0, microsecond=0
+            )
+        )
+    ).total_seconds()
+    if diff >= 60:
+        return True
+    elif diff >= 0:
+        return False
+
+    raise ValueError(
+        "Check reach the next minute function should check a datetime that not "
+        "less than the current date"
+    )
+
+
+def wait_to_next_minute(
+    dt: datetime, second: float = 0
+) -> None:  # pragma: no cov
     """Wait with sleep to the next minute with an offset second value."""
-    future = now.replace(second=0, microsecond=0) + timedelta(minutes=1)
-    time.sleep((future - now).total_seconds() + second)
+    future = dt.replace(second=0, microsecond=0) + timedelta(minutes=1)
+    time.sleep((future - dt).total_seconds() + second)
 
 
 def delay(second: float = 0) -> None:  # pragma: no cov
@@ -92,6 +116,8 @@ def gen_id(
 
     :rtype: str
     """
+    from .conf import config
+
     if not isinstance(value, str):
         value: str = str(value)
 
@@ -177,7 +203,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
     """Batch data into iterators of length n. The last batch may be shorter.
 
     Example:
-        >>> for b in batch('ABCDEFG', 3):
+        >>> for b in batch(iter('ABCDEFG'), 3):
         ...     print(list(b))
         ['A', 'B', 'C']
         ['D', 'E', 'F']

ddeutil/workflow/workflow.py

@@ -29,7 +29,8 @@ from concurrent.futures import (
 )
 from dataclasses import field
 from datetime import datetime, timedelta
-from functools import total_ordering
+from enum import Enum
+from functools import partial, total_ordering
 from heapq import heappop, heappush
 from queue import Queue
 from textwrap import dedent
@@ -53,21 +54,33 @@ from .utils import (
     cut_id,
     gen_id,
     get_dt_now,
-    wait_a_minute,
+    reach_next_minute,
+    wait_to_next_minute,
 )
 
 logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
-    "Workflow",
     "Release",
     "ReleaseQueue",
+    "ReleaseType",
+    "Workflow",
     "WorkflowTask",
 )
 
 
+class ReleaseType(str, Enum):
+    """Release Type Enum support the type field on the Release dataclass."""
+
+    DEFAULT: str = "manual"
+    TASK: str = "task"
+    POKE: str = "poking"
+
+
 @total_ordering
-@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+@dataclass(
+    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+)
 class Release:
     """Release Pydantic dataclass object that use for represent
     the release data that use with the `workflow.release` method."""
@@ -76,7 +89,7 @@ class Release:
     offset: float
     end_date: datetime
     runner: CronRunner
-    type: str
+    type: ReleaseType = field(default=ReleaseType.DEFAULT)
 
     def __repr__(self) -> str:
         return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
@@ -94,13 +107,16 @@ class Release:
         """
         if isinstance(dt, str):
             dt: datetime = datetime.fromisoformat(dt)
+        elif not isinstance(dt, datetime):
+            raise TypeError(
+                "The `from_dt` need argument type be str or datetime only."
+            )
 
         return cls(
             date=dt,
             offset=0,
             end_date=dt + timedelta(days=1),
             runner=CronJob("* * * * *").schedule(dt.replace(tzinfo=config.tz)),
-            type="manual",
         )
 
     def __eq__(self, other: Release | datetime) -> bool:
@@ -174,13 +190,6 @@ class ReleaseQueue:
 
         :rtype: bool
         """
-        # NOTE: Old logic to peeking the first release from waiting queue.
-        #
-        # first_value: Release = heappop(self.queue)
-        # heappush(self.queue, first_value)
-        #
-        # return first_value
-        #
         return self.queue[0]
 
     def check_queue(self, value: Release | datetime) -> bool:
@@ -477,7 +486,7 @@ class Workflow(BaseModel):
         *,
         run_id: str | None = None,
         log: type[Log] = None,
-        queue: ReleaseQueue | list[datetime] | list[Release] | None = None,
+        queue: ReleaseQueue | None = None,
         override_log_name: str | None = None,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
@@ -487,7 +496,7 @@ class Workflow(BaseModel):
             This method allow workflow use log object to save the execution
         result to log destination like file log to the local `/logs` directory.
 
-        :Steps:
+        Steps:
             - Initialize ReleaseQueue and Release if they do not pass.
             - Create release data for pass to parameter templating function.
             - Execute this workflow with mapping release data to its parameters.
@@ -497,7 +506,7 @@ class Workflow(BaseModel):
 
         :param release: A release datetime or Release object.
         :param params: A workflow parameter that pass to execute method.
-        :param queue: A list of release time that already queue.
+        :param queue: A ReleaseQueue that use for mark complete.
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
         :param queue: A ReleaseQueue object.
@@ -510,13 +519,14 @@ class Workflow(BaseModel):
         name: str = override_log_name or self.name
         run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
-        rs_release_type: str = "release"
 
-        # VALIDATE: Change queue value to ReleaseQueue object.
-        if queue is None or isinstance(queue, list):
-            queue: ReleaseQueue = ReleaseQueue.from_list(queue)
+        if queue is not None and not isinstance(queue, ReleaseQueue):
+            raise TypeError(
+                "The queue argument should be ReleaseQueue object only."
+            )
 
         # VALIDATE: Change release value to Release object.
+        rs_release_type: str = "release"
         if isinstance(release, datetime):
             rs_release_type: str = "datetime"
             release: Release = Release.from_dt(release)
@@ -548,24 +558,26 @@ class Workflow(BaseModel):
         )
 
         rs.set_parent_run_id(run_id)
-        rs_log: Log = log.model_validate(
-            {
-                "name": name,
-                "release": release.date,
-                "type": release.type,
-                "context": rs.context,
-                "parent_run_id": rs.parent_run_id,
-                "run_id": rs.run_id,
-            }
-        )
 
         # NOTE: Saving execution result to destination of the input log object.
         logger.debug(f"({cut_id(run_id)}) [LOG]: Writing log: {name!r}.")
-        rs_log.save(excluded=None)
+        (
+            log.model_validate(
+                {
+                    "name": name,
+                    "release": release.date,
+                    "type": release.type,
+                    "context": rs.context,
+                    "parent_run_id": rs.parent_run_id,
+                    "run_id": rs.run_id,
+                }
+            ).save(excluded=None)
+        )
 
         # NOTE: Remove this release from running.
-        queue.remove_running(release)
-        queue.mark_complete(release)
+        if queue is not None:
+            queue.remove_running(release)
+            queue.mark_complete(release)
 
         # NOTE: Remove the params key from the result context for deduplicate.
         context: dict[str, Any] = rs.context
@@ -594,8 +606,16 @@ class Workflow(BaseModel):
         *,
         force_run: bool = False,
     ) -> ReleaseQueue:
-        """Generate queue of datetime from the cron runner that initialize from
-        the on field. with offset value.
+        """Generate Release from all on values from the on field and store them
+        to the ReleaseQueue object.
+
+        Steps:
+            - For-loop all the on value in the on field.
+            - Create Release object from the current date that not reach the end
+              date.
+            - Check this release do not store on the release queue object.
+              Generate the next date if it exists.
+            - Push this release to the release queue
 
         :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
@@ -621,7 +641,7 @@ class Workflow(BaseModel):
                 offset=offset,
                 end_date=end_date,
                 runner=runner,
-                type="poking",
+                type=ReleaseType.POKE,
             )
 
             while queue.check_queue(workflow_release) or (
@@ -633,7 +653,7 @@ class Workflow(BaseModel):
                     offset=offset,
                     end_date=end_date,
                     runner=runner,
-                    type="poking",
+                    type=ReleaseType.POKE,
                 )
 
             if runner.date > end_date:
@@ -662,6 +682,9 @@ class Workflow(BaseModel):
             This method will observe its schedule that nearing to run with the
         ``self.release()`` method.
 
+            The limitation of this method is not allow run a date that less
+        than the current date.
+
         :param start_date: A start datetime object.
         :param params: A parameters that want to pass to the release method.
         :param run_id: A workflow running ID for this poke.
@@ -678,6 +701,12 @@ class Workflow(BaseModel):
         log: type[Log] = log or get_log()
         run_id: str = run_id or gen_id(self.name, unique=True)
 
+        # VALIDATE: Check the periods value should gather than 0.
+        if periods <= 0:
+            raise WorkflowException(
+                "The period of poking should be int and grater or equal than 1."
+            )
+
         # NOTE: If this workflow does not set the on schedule, it will return
         #   empty result.
         if len(self.on) == 0:
@@ -687,23 +716,25 @@ class Workflow(BaseModel):
             )
             return []
 
-        if periods <= 0:
-            raise WorkflowException(
-                "The period of poking should be int and grater or equal than 1."
-            )
+        # NOTE: Create the current date that change microsecond to 0
+        current_date: datetime = datetime.now(tz=config.tz).replace(
+            microsecond=0
+        )
 
         # NOTE: Create start_date and offset variables.
-        current_date: datetime = datetime.now(tz=config.tz)
-
         if start_date and start_date <= current_date:
-            start_date = start_date.replace(tzinfo=config.tz)
+            start_date = start_date.replace(tzinfo=config.tz).replace(
+                microsecond=0
+            )
             offset: float = (current_date - start_date).total_seconds()
         else:
+            # NOTE: Force change start date if it gathers than the current date,
+            #   or it does not pass to this method.
             start_date: datetime = current_date
             offset: float = 0
 
-        # NOTE: End date is using to stop generate queue with an input periods
-        #   value.
+        # NOTE: The end date is using to stop generate queue with an input
+        #   periods value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
         logger.info(
@@ -715,17 +746,17 @@ class Workflow(BaseModel):
         results: list[Result] = []
 
         # NOTE: Create empty ReleaseQueue object.
-        wf_queue: ReleaseQueue = ReleaseQueue()
+        q: ReleaseQueue = ReleaseQueue()
 
-        # NOTE: Make queue to the workflow queue object.
-        self.queue(
-            offset,
-            end_date=end_date,
-            queue=wf_queue,
-            log=log,
-            force_run=force_run,
+        # NOTE: Create reusable partial function and add Release to the release
+        #   queue object.
+        partial_queue = partial(
+            self.queue, offset, end_date, log=log, force_run=force_run
         )
-        if not wf_queue.is_queued:
+        partial_queue(q)
+
+        # NOTE: Return the empty result if it does not have any Release.
+        if not q.is_queued:
             logger.info(
                 f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have "
                 f"any queue."
@@ -741,34 +772,27 @@ class Workflow(BaseModel):
 
             futures: list[Future] = []
 
-            while wf_queue.is_queued:
+            while q.is_queued:
 
-                # NOTE: Pop the latest Release object from queue.
-                release: Release = heappop(wf_queue.queue)
+                # NOTE: Pop the latest Release object from the release queue.
+                release: Release = heappop(q.queue)
 
-                if (
-                    release.date - get_dt_now(tz=config.tz, offset=offset)
-                ).total_seconds() > 60:
+                if reach_next_minute(release.date, tz=config.tz, offset=offset):
                     logger.debug(
-                        f"({cut_id(run_id)}) [POKING]: Wait because the latest "
-                        f"release has diff time more than 60 seconds ..."
+                        f"({cut_id(run_id)}) [POKING]: The latest release, "
+                        f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
+                        f"on this minute"
                     )
-                    heappush(wf_queue.queue, release)
-                    wait_a_minute(get_dt_now(tz=config.tz, offset=offset))
+                    heappush(q.queue, release)
+                    wait_to_next_minute(get_dt_now(tz=config.tz, offset=offset))
 
                     # WARNING: I already call queue poking again because issue
                     #   about the every minute crontab.
-                    self.queue(
-                        offset,
-                        end_date,
-                        queue=wf_queue,
-                        log=log,
-                        force_run=force_run,
-                    )
+                    partial_queue(q)
                     continue
 
                 # NOTE: Push the latest Release to the running queue.
-                heappush(wf_queue.running, release)
+                heappush(q.running, release)
 
                 futures.append(
                     executor.submit(
@@ -776,17 +800,11 @@ class Workflow(BaseModel):
                         release=release,
                         params=params,
                         log=log,
-                        queue=wf_queue,
+                        queue=q,
                     )
                 )
 
-                self.queue(
-                    offset,
-                    end_date,
-                    queue=wf_queue,
-                    log=log,
-                    force_run=force_run,
-                )
+                partial_queue(q)
 
             # WARNING: This poking method does not allow to use fail-fast
             #   logic to catching parallel execution result.
@@ -1148,20 +1166,31 @@ class WorkflowTask:
         release: datetime | Release | None = None,
         run_id: str | None = None,
         log: type[Log] = None,
-        queue: ReleaseQueue | list[datetime] | list[Release] | None = None,
+        queue: ReleaseQueue | None = None,
     ) -> Result:
         """Release the workflow task data.
 
         :param release: A release datetime or Release object.
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
-        :param queue: A ReleaseQueue object.
+        :param queue: A ReleaseQueue object that use to mark complete.
 
         :rtype: Result
         """
         log: type[Log] = log or get_log()
 
         if release is None:
+
+            if queue is None:
+                raise ValueError(
+                    "If pass None release value, you should to pass the queue"
+                    "for generate this release."
+                )
+            elif not isinstance(queue, ReleaseQueue):
+                raise TypeError(
+                    "The queue argument should be ReleaseQueue object only."
+                )
+
             if queue.check_queue(self.runner.date):
                 release = self.runner.next
 
@@ -1170,6 +1199,7 @@ class WorkflowTask:
             else:
                 release = self.runner.date
 
+        # NOTE: Call the workflow release method.
         return self.workflow.release(
             release=release,
             params=self.values,
@@ -1186,8 +1216,9 @@ class WorkflowTask:
         log: type[Log],
         *,
         force_run: bool = False,
-    ):
-        """Generate Release to ReleaseQueue object.
+    ) -> ReleaseQueue:
+        """Generate Release from the runner field and store it to the
+        ReleaseQueue object.
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
@@ -1205,7 +1236,7 @@ class WorkflowTask:
             offset=0,
             end_date=end_date,
             runner=self.runner,
-            type="task",
+            type=ReleaseType.TASK,
         )
 
         while queue.check_queue(workflow_release) or (
@@ -1217,7 +1248,7 @@ class WorkflowTask:
                 offset=0,
                 end_date=end_date,
                 runner=self.runner,
-                type="task",
+                type=ReleaseType.TASK,
             )
 
         if self.runner.date > end_date:

{ddeutil_workflow-0.0.30.dist-info → ddeutil_workflow-0.0.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.30
+Version: 0.0.31
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -192,6 +192,15 @@ The above workflow template is main executor pipeline that you want to do. If yo
 want to schedule this workflow, you want to dynamic its parameters change base on
 execution time such as `run-date` should change base on that workflow running date.
 
+```python
+from ddeutil.workflow import Workflow, Result
+
+workflow: Workflow = Workflow.from_loader('run-py-local')
+result: Result = workflow.execute(
+   params={"source-extract": "USD-THB", "asat-dt": "2024-01-01"}
+)
+```
+
 So, this package provide the `Schedule` template for this action.
 
 ```yaml

ddeutil_workflow-0.0.31.dist-info/RECORD

@@ -0,0 +1,25 @@
+ddeutil/workflow/__about__.py,sha256=3Fv8LNxLB3UmoMSQqDZv556u3eR-OYBH8pw9kM6FeE4,28
+ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
+ddeutil/workflow/__init__.py,sha256=pRIZIGwC7Xs8Ur7-jHPIAMLriD5If9zOPc-ZmKZS2XQ,1678
+ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
+ddeutil/workflow/conf.py,sha256=6yGbSi69lsccYgnrwTzdjdPhU54hUop2e1GjBNres08,17663
+ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
+ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
+ddeutil/workflow/hook.py,sha256=MgZFlTGvaRSBrTouZGlxwYpKQoKDOT26PNhESeL3LY0,5469
+ddeutil/workflow/job.py,sha256=XcewyALsLYYq94ycF6mkj3Ydr6if683z7t1oBqEVInE,24290
+ddeutil/workflow/params.py,sha256=AJLiTaF6lG37SvzyniTCug5-TgZTmNCn5fJXb-CCcqM,5707
+ddeutil/workflow/result.py,sha256=8LItqF-Xe6pAAWkAsY_QFkKBOA0fEBh97I2og3CZsPc,3409
+ddeutil/workflow/scheduler.py,sha256=2Y_ewAP1iQKgD81i6H6fXuJLCVqLocumEmCG2SomEqg,22214
+ddeutil/workflow/stage.py,sha256=glAhvgvyQ98n2JLUBU8MUs48FEPkfU1fvh3Wwi9PSCg,24293
+ddeutil/workflow/templates.py,sha256=bVU_8gnMQmdhhw3W28ZqwmpEaOx10Nx_aauqiLS0lqg,10807
+ddeutil/workflow/utils.py,sha256=rTDQKaaber7cRqzJjWpCP9OTbarti1UMKdLgH6VRjFM,6709
+ddeutil/workflow/workflow.py,sha256=ET1otR5VcfnOMoNiW7EMb1_wIaxNw9yWsBXS5kVWG9s,43428
+ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
+ddeutil/workflow/api/api.py,sha256=Ma9R8yuQAhowG_hox-k53swFsf5IAvheEnSxNQ-8DaQ,4039
+ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
+ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
+ddeutil_workflow-0.0.31.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.31.dist-info/METADATA,sha256=-cDcCfcV-4_dU_91GSy5sdV-ZkEWbdk7wDmAJB24puU,15090
+ddeutil_workflow-0.0.31.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ddeutil_workflow-0.0.31.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.31.dist-info/RECORD,,

ddeutil_workflow-0.0.30.dist-info/RECORD

@@ -1,25 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=zSy9Xk11PPZkgJ5Db1_kZp2yzt5inkEHVJWwFHPWlmk,28
-ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
-ddeutil/workflow/__init__.py,sha256=dghn2lFl3Own4Pyq7SFHu-FMymOgLontJ6aCfxea9h4,1606
-ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
-ddeutil/workflow/conf.py,sha256=7lj_Im9jsa95fWUo19Q4-ZAcHa8Pu1HW-vaLgvrjNUM,17559
-ddeutil/workflow/cron.py,sha256=OLgniUxmrn65gzckk-uTmE2Pk1enJJyjYUKVeBbDQz0,7522
-ddeutil/workflow/exceptions.py,sha256=XUnpJSuxOyataClP0w_gpYjzn-NIwZK2BHro-J7Yw24,895
-ddeutil/workflow/hook.py,sha256=MgZFlTGvaRSBrTouZGlxwYpKQoKDOT26PNhESeL3LY0,5469
-ddeutil/workflow/job.py,sha256=XcewyALsLYYq94ycF6mkj3Ydr6if683z7t1oBqEVInE,24290
-ddeutil/workflow/params.py,sha256=svCjmFgEhim8yFJVjZhFmKP8JqTDHQ5EPhwJHVuDGno,5289
-ddeutil/workflow/result.py,sha256=k4pcj5KjbEcEPymsEUXeGY4gyLMfPkMTO6YDrAtfk7Q,3408
-ddeutil/workflow/scheduler.py,sha256=f3d7c5SVgY5Q1JsHQ6cH513CJmJkh4l8YcKAGYudJRc,20426
-ddeutil/workflow/stage.py,sha256=wn8CARTvFJY4ZK1SwjzH8sKoMRz_eIeSGUMgnDWNi6g,24031
-ddeutil/workflow/templates.py,sha256=bVU_8gnMQmdhhw3W28ZqwmpEaOx10Nx_aauqiLS0lqg,10807
-ddeutil/workflow/utils.py,sha256=8LTqpvRPfrEYxsxhwszk6GKkyjrswxnwF3r_9vE8szw,6059
-ddeutil/workflow/workflow.py,sha256=ZLbG-K2gSNAsDGiHPjbtJd7rsEFf6jfVGAVB_9jpy84,42103
-ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
-ddeutil/workflow/api/api.py,sha256=Ma9R8yuQAhowG_hox-k53swFsf5IAvheEnSxNQ-8DaQ,4039
-ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
-ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
-ddeutil_workflow-0.0.30.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.30.dist-info/METADATA,sha256=zbVHOL41qpFRG83SacZVYK9tS2JRTCM61cpnSXty9LU,14868
-ddeutil_workflow-0.0.30.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ddeutil_workflow-0.0.30.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.30.dist-info/RECORD,,