ddeutil-workflow 0.0.30__py3-none-any.whl → 0.0.32__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -33,7 +33,7 @@ from functools import wraps
 from heapq import heappop, heappush
 from textwrap import dedent
 from threading import Thread
-from typing import Callable, Optional, TypedDict
+from typing import Callable, Optional, TypedDict, Union
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import field_validator, model_validator
@@ -41,7 +41,7 @@ from typing_extensions import Self
 
 try:
     from typing import ParamSpec
-except ImportError:
+except ImportError:  # pragma: no cov
     from typing_extensions import ParamSpec
 
 try:
@@ -53,11 +53,9 @@ from .__cron import CronRunner
 from .__types import DictData, TupleStr
 from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
-from .exceptions import WorkflowException
-from .utils import (
-    batch,
-    delay,
-)
+from .exceptions import ScheduleException, WorkflowException
+from .result import Result
+from .utils import batch, delay
 from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
 
 P = ParamSpec("P")
@@ -69,7 +67,7 @@ logging.getLogger("schedule").setLevel(logging.INFO)
 
 __all__: TupleStr = (
     "Schedule",
-    "WorkflowSchedule",
+    "ScheduleWorkflow",
     "schedule_task",
     "monitor",
     "schedule_control",
@@ -79,8 +77,8 @@ __all__: TupleStr = (
 )
 
 
-class WorkflowSchedule(BaseModel):
-    """Workflow Schedule Pydantic model that use to keep workflow model for
+class ScheduleWorkflow(BaseModel):
+    """Schedule Workflow Pydantic model that use to keep workflow model for
     the Schedule model. it should not use Workflow model directly because on the
     schedule config it can adjust crontab value that different from the Workflow
     model.
@@ -233,9 +231,9 @@ class Schedule(BaseModel):
             "A schedule description that can be string of markdown content."
         ),
     )
-    workflows: list[WorkflowSchedule] = Field(
+    workflows: list[ScheduleWorkflow] = Field(
         default_factory=list,
-        description="A list of WorkflowSchedule models.",
+        description="A list of ScheduleWorkflow models.",
     )
 
     @field_validator("desc", mode="after")
@@ -258,7 +256,7 @@ class Schedule(BaseModel):
         an input schedule name. The loader object will use this schedule name to
         searching configuration data of this schedule model in conf path.
 
-        :param name: A schedule name that want to pass to Loader object.
+        :param name: (str) A schedule name that want to pass to Loader object.
         :param externals: An external parameters that want to pass to Loader
             object.
 
@@ -310,8 +308,102 @@ class Schedule(BaseModel):
 
         return workflow_tasks
 
+    def pending(
+        self,
+        *,
+        stop: datetime | None = None,
+        externals: DictData | None = None,
+        log: type[Log] | None = None,
+    ) -> None:  # pragma: no cov
+        """Pending this schedule tasks with the schedule package.
+
+        :param stop: A datetime value that use to stop running schedule.
+        :param externals: An external parameters that pass to Loader.
+        :param log: A log class that use on the workflow task release for
+            writing its release log context.
+        """
+        try:
+            from schedule import Scheduler
+        except ImportError:
+            raise ImportError(
+                "Should install schedule package before use this method."
+            ) from None
+
+        # NOTE: Get default logging.
+        log: type[Log] = log or get_log()
+        scheduler: Scheduler = Scheduler()
+
+        # NOTE: Create the start and stop datetime.
+        start_date: datetime = datetime.now(tz=config.tz)
+        stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+
+        # IMPORTANT: Create main mapping of queue and thread object.
+        queue: dict[str, ReleaseQueue] = {}
+        threads: ReleaseThreads = {}
 
-ReturnCancelJob = Callable[P, Optional[CancelJob]]
+        start_date_waiting: datetime = start_date.replace(
+            second=0, microsecond=0
+        ) + timedelta(minutes=1)
+
+        # NOTE: This schedule job will start every minute at :02 seconds.
+        (
+            scheduler.every(1)
+            .minutes.at(":02")
+            .do(
+                schedule_task,
+                tasks=self.tasks(
+                    start_date_waiting, queue=queue, externals=externals
+                ),
+                stop=stop_date,
+                queue=queue,
+                threads=threads,
+                log=log,
+            )
+            .tag("control")
+        )
+
+        # NOTE: Checking zombie task with schedule job will start every 5 minute at
+        #   :10 seconds.
+        (
+            scheduler.every(5)
+            .minutes.at(":10")
+            .do(
+                monitor,
+                threads=threads,
+            )
+            .tag("monitor")
+        )
+
+        # NOTE: Start running schedule
+        logger.info(
+            f"[SCHEDULE]: Schedule with stopper: {stop_date:%Y-%m-%d %H:%M:%S}"
+        )
+
+        while True:
+            scheduler.run_pending()
+            time.sleep(1)
+
+            # NOTE: Break the scheduler when the control job does not exist.
+            if not scheduler.get_jobs("control"):
+                scheduler.clear("monitor")
+
+                while len(threads) > 0:
+                    logger.warning(
+                        "[SCHEDULE]: Waiting schedule release thread that still "
+                        "running in background."
+                    )
+                    delay(10)
+                    monitor(threads)
+
+                break
+
+        logger.warning(
+            f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
+        )
+
+
+ResultOrCancelJob = Union[type[CancelJob], Result]
+ReturnCancelJob = Callable[P, ResultOrCancelJob]
 DecoratorCancelJob = Callable[[ReturnCancelJob], ReturnCancelJob]
 
 
@@ -326,24 +418,25 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
     """
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:  # pragma: no cov
-        try:
 
-            @wraps(func)
-            def wrapper(*args, **kwargs):
+        @wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> ResultOrCancelJob:
+            try:
                 return func(*args, **kwargs)
+            except Exception as err:
+                logger.exception(err)
+                if cancel_on_failure:
+                    return CancelJob
+                raise err
 
-            return wrapper
-
-        except Exception as err:
-            logger.exception(err)
-            if cancel_on_failure:
-                return CancelJob
-            raise err
+        return wrapper
 
     return decorator
 
 
 class ReleaseThread(TypedDict):
+    """TypeDict for the release thread."""
+
     thread: Thread
     start_date: datetime
 
@@ -358,11 +451,13 @@ def schedule_task(
     queue: dict[str, ReleaseQueue],
     threads: ReleaseThreads,
     log: type[Log],
-) -> CancelJob | None:
-    """Workflow task generator that create release pair of workflow and on to
-    the threading in background.
+) -> type[CancelJob] | None:
+    """Schedule task function that generate thread of workflow task release
+    method in background. This function do the same logic as the workflow poke
+    method, but it runs with map of schedules and the on values.
 
-        This workflow task will start every minute at ':02' second.
+        This schedule task start runs every minute at ':02' second, and it does
+    not allow you to run with offset time.
 
     :param tasks: A list of WorkflowTask object.
     :param stop: A stop datetime object that force stop running scheduler.
@@ -370,7 +465,7 @@ def schedule_task(
     :param threads: A mapping of alias name and Thread object.
     :param log: A log class that want to make log object.
 
-    :rtype: CancelJob | None
+    :rtype: type[CancelJob] | None
     """
     current_date: datetime = datetime.now(tz=config.tz)
     if current_date > stop.replace(tzinfo=config.tz):
@@ -381,15 +476,16 @@ def schedule_task(
     #   function. It will deplicate running with different schedule value
     #   because I use current time in this condition.
     #
-    #       For example, if a workflow A queue has '00:02:00' time that
-    #   should to run and its schedule has '*/2 * * * *' and '*/35 * * * *'.
-    #   This condition will release with 2 threading job.
+    #       For example, if a queue has a time release be '00:02:00' that should
+    #   to run and its schedule has '*/2 * * * *' and '*/35 * * * *'.
+    #   This condition make this function create 2 threading tasks.
     #
-    #   '00:02:00'  --> '*/2 * * * *'   --> running
-    #               --> '*/35 * * * *'  --> skip
+    #       '00:02:00'  --> '*/2 * * * *'   --> run
+    #                   --> '*/35 * * * *'  --> skip
     #
     for task in tasks:
 
+        # NOTE: Get the ReleaseQueue with an alias of the WorkflowTask.
         q: ReleaseQueue = queue[task.alias]
 
         # NOTE: Start adding queue and move the runner date in the WorkflowTask.
@@ -410,12 +506,17 @@ def schedule_task(
         current_release: datetime = current_date.replace(
             second=0, microsecond=0
         )
-        if (first_date := q.first_queue.date) != current_release:
+        if (first_date := q.first_queue.date) > current_release:
             logger.debug(
                 f"[WORKFLOW]: Skip schedule "
                 f"{first_date:%Y-%m-%d %H:%M:%S} for : {task.alias!r}"
             )
             continue
+        elif first_date < current_release:  # pragma: no cov
+            raise ScheduleException(
+                "The first release date from queue should not less than current"
+                "release date."
+            )
 
         # NOTE: Pop the latest release and push it to running.
         release: Release = heappop(q.queue)
@@ -445,7 +546,7 @@ def schedule_task(
 
         delay()
 
-    logger.debug(f"[SCHEDULE]: End schedule release {'=' * 80}")
+    logger.debug(f"[SCHEDULE]: End schedule task {'=' * 80}")
 
 
 def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
@@ -455,9 +556,7 @@ def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
     :param threads: A mapping of Thread object and its name.
     :type threads: ReleaseThreads
     """
-    logger.debug(
-        "[MONITOR]: Start checking long running workflow release task."
-    )
+    logger.debug("[MONITOR]: Start checking long running schedule task.")
 
     snapshot_threads: list[str] = list(threads.keys())
     for t_name in snapshot_threads:
@@ -476,12 +575,15 @@ def schedule_control(
     *,
     log: type[Log] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Scheduler control function that running every minute.
+    """Scheduler control function that run the chuck of schedules every minute
+    and this function release monitoring thread for tracking undead thread in
+    the background.
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: A datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
-    :param log:
+    :param log: A log class that use on the workflow task release for writing
+        its release log context.
 
     :rtype: list[str]
     """
@@ -493,8 +595,11 @@ def schedule_control(
             "Should install schedule package before use this module."
         ) from None
 
+    # NOTE: Get default logging.
     log: type[Log] = log or get_log()
     scheduler: Scheduler = Scheduler()
+
+    # NOTE: Create the start and stop datetime.
     start_date: datetime = datetime.now(tz=config.tz)
     stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
 
@@ -506,7 +611,6 @@ def schedule_control(
         second=0, microsecond=0
     ) + timedelta(minutes=1)
 
-    # NOTE: Start create workflow tasks from list of schedule name.
     tasks: list[WorkflowTask] = []
     for name in schedules:
         schedule: Schedule = Schedule.from_loader(name, externals=externals)
@@ -533,7 +637,8 @@ def schedule_control(
         .tag("control")
     )
 
-    # NOTE: Checking zombie task with schedule job will start every 5 minute.
+    # NOTE: Checking zombie task with schedule job will start every 5 minute at
+    #   :10 seconds.
     (
         scheduler.every(5)
         .minutes.at(":10")
@@ -563,7 +668,7 @@ def schedule_control(
                     "[SCHEDULE]: Waiting schedule release thread that still "
                     "running in background."
                 )
-                delay(15)
+                delay(10)
                 monitor(threads)
 
             break
@@ -579,16 +684,15 @@ def schedule_runner(
     externals: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Schedule runner function for start submit the ``schedule_control`` func
-    in multiprocessing pool with chunk of schedule config that exists in config
-    path by ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS``.
+    """Schedule runner function it the multiprocess controller function for
+    split the setting schedule to the `schedule_control` function on the
+    process pool. It chunks schedule configs that exists in config
+    path by `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` value.
 
     :param stop: A stop datetime object that force stop running scheduler.
     :param externals:
     :param excluded: A list of schedule name that want to exclude from finding.
 
-    :rtype: list[str]
-
         This function will get all workflows that include on value that was
     created in config path and chuck it with application config variable
     ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS`` env var to multiprocess executor
@@ -600,7 +704,9 @@ def schedule_runner(
                                         --> thread of release task 01 02
                             ==> schedule --> thread of release task 02 01
                                         --> thread of release task 02 02
-            ==> process 02
+            ==> process 02  ==> ...
+
+    :rtype: list[str]
     """
     results: list[str] = []
 

ddeutil/workflow/stage.py

@@ -328,7 +328,7 @@ class BashStage(BaseStage):
     If your current OS is Windows, it will run on the bash in the WSL.
 
         I get some limitation when I run shell statement with the built-in
-    supprocess package. It does not good enough to use multiline statement.
+    subprocess package. It does not good enough to use multiline statement.
     Thus, I add writing ``.sh`` file before execution process for fix this
     issue.
 
@@ -665,3 +665,15 @@ Stage = Union[
     TriggerStage,
     EmptyStage,
 ]
+
+
+# TODO: Not implement this stages yet
+class ParallelStage(BaseModel):  # pragma: no cov
+    parallel: list[Stage]
+    max_parallel_core: int = Field(default=2)
+
+
+# TODO: Not implement this stages yet
+class ForEachStage(BaseModel):  # pragma: no cov
+    foreach: list[str]
+    stages: list[Stage]

ddeutil/workflow/templates.py

@@ -79,7 +79,7 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
 def make_filter_registry() -> dict[str, FilterRegistry]:
     """Return registries of all functions that able to called with task.
 
-    :rtype: dict[str, Registry]
+    :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
     for module in config.regis_filter:
@@ -108,6 +108,8 @@ def get_args_const(
 ) -> tuple[str, list[Constant], dict[str, Constant]]:
     """Get arguments and keyword-arguments from function calling string.
 
+    :param expr: An expr string value.
+
     :rtype: tuple[str, list[Constant], dict[str, Constant]]
     """
     try:
@@ -150,6 +152,11 @@ def get_args_from_filter(
 ) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]:  # pragma: no cov
     """Get arguments and keyword-arguments from filter function calling string.
     and validate it with the filter functions mapping dict.
+
+    :param ft:
+    :param filters:
+
+    :rtype: tuple[str, FilterRegistry, list[Any], dict[Any, Any]]
     """
     func_name, _args, _kwargs = get_args_const(ft)
     args: list[Any] = [arg.value for arg in _args]
@@ -243,7 +250,7 @@ def str2template(
     params: DictData,
     *,
     filters: dict[str, FilterRegistry] | None = None,
-) -> Any:
+) -> str:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
 
@@ -255,6 +262,8 @@ def str2template(
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters:
+
+    :rtype: str
     """
     filters: dict[str, FilterRegistry] = filters or make_filter_registry()
 
@@ -295,7 +304,7 @@ def str2template(
     return search_env_replace(value)
 
 
-def param2template(value: Any, params: DictData) -> Any:
+def param2template(value: T, params: DictData) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
@@ -303,7 +312,7 @@ def param2template(value: Any, params: DictData) -> Any:
     :param params: A parameter value that getting with matched regular
         expression.
 
-    :rtype: Any
+    :rtype: T
     :returns: An any getter value from the params input.
     """
     filters: dict[str, FilterRegistry] = make_filter_registry()

ddeutil/workflow/utils.py

@@ -21,10 +21,9 @@ from zoneinfo import ZoneInfo
 from ddeutil.core import hash_str
 
 from .__types import DictData, Matrix
-from .conf import config
 
 T = TypeVar("T")
-
+UTC = ZoneInfo("UTC")
 logger = logging.getLogger("ddeutil.workflow")
 
 
@@ -37,7 +36,7 @@ def get_dt_now(
     :param offset:
     :return: The current datetime object that use an input timezone or UTC.
     """
-    return datetime.now(tz=(tz or ZoneInfo("UTC"))) - timedelta(seconds=offset)
+    return datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)
 
 
 def get_diff_sec(
@@ -52,17 +51,42 @@ def get_diff_sec(
     """
     return round(
         (
-            dt
-            - datetime.now(tz=(tz or ZoneInfo("UTC")))
-            - timedelta(seconds=offset)
+            dt - datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)
         ).total_seconds()
     )
 
 
-def wait_a_minute(now: datetime, second: float = 2) -> None:  # pragma: no cov
+def reach_next_minute(
+    dt: datetime, tz: ZoneInfo | None = None, offset: float = 0.0
+) -> bool:
+    """Check this datetime object is not in range of minute level on the current
+    datetime.
+    """
+    diff: float = (
+        dt.replace(second=0, microsecond=0)
+        - (
+            get_dt_now(tz=(tz or UTC), offset=offset).replace(
+                second=0, microsecond=0
+            )
+        )
+    ).total_seconds()
+    if diff >= 60:
+        return True
+    elif diff >= 0:
+        return False
+
+    raise ValueError(
+        "Check reach the next minute function should check a datetime that not "
+        "less than the current date"
+    )
+
+
+def wait_to_next_minute(
+    dt: datetime, second: float = 0
+) -> None:  # pragma: no cov
     """Wait with sleep to the next minute with an offset second value."""
-    future = now.replace(second=0, microsecond=0) + timedelta(minutes=1)
-    time.sleep((future - now).total_seconds() + second)
+    future = dt.replace(second=0, microsecond=0) + timedelta(minutes=1)
+    time.sleep((future - dt).total_seconds() + second)
 
 
 def delay(second: float = 0) -> None:  # pragma: no cov
@@ -92,6 +116,8 @@ def gen_id(
 
     :rtype: str
     """
+    from .conf import config
+
     if not isinstance(value, str):
         value: str = str(value)
 
@@ -177,7 +203,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
     """Batch data into iterators of length n. The last batch may be shorter.
 
     Example:
-        >>> for b in batch('ABCDEFG', 3):
+        >>> for b in batch(iter('ABCDEFG'), 3):
         ...     print(list(b))
         ['A', 'B', 'C']
         ['D', 'E', 'F']