ddeutil-workflow 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 """
-The main schedule running is ``workflow_runner`` function that trigger the
+The main schedule running is ``schedule_runner`` function that trigger the
 multiprocess of ``workflow_control`` function for listing schedules on the
 config by ``Loader.finds(Schedule)``.
 
@@ -21,7 +21,6 @@ value with the on field.
 from __future__ import annotations
 
 import copy
-import inspect
 import logging
 import time
 from concurrent.futures import (
@@ -31,9 +30,10 @@ from concurrent.futures import (
 )
 from datetime import datetime, timedelta
 from functools import wraps
+from heapq import heappop, heappush
 from textwrap import dedent
 from threading import Thread
-from typing import Callable, Optional
+from typing import Callable, Optional, TypedDict
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import field_validator, model_validator
@@ -51,30 +51,31 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, config, get_logger
+from .conf import FileLog, Loader, Log, config, get_logger
+from .cron import On
 from .exceptions import WorkflowException
-from .on import On
 from .utils import (
     batch,
     delay,
-    queue2str,
 )
-from .workflow import Workflow, WorkflowTaskData
+from .workflow import Workflow, WorkflowQueue, WorkflowRelease, WorkflowTask
 
 P = ParamSpec("P")
 logger = get_logger("ddeutil.workflow")
 
-# NOTE: Adjust logging level on the schedule package.
+# NOTE: Adjust logging level on the `schedule` package.
 logging.getLogger("schedule").setLevel(logging.INFO)
 
 
 __all__: TupleStr = (
     "Schedule",
     "WorkflowSchedule",
-    "workflow_task_release",
-    "workflow_monitor",
-    "workflow_control",
-    "workflow_runner",
+    "schedule_task",
+    "monitor",
+    "schedule_control",
+    "schedule_runner",
+    "ReleaseThreads",
+    "ReleaseThread",
 )
 
 
@@ -166,6 +167,58 @@ class WorkflowSchedule(BaseModel):
 
         return value
 
+    def tasks(
+        self,
+        start_date: datetime,
+        queue: dict[str, WorkflowQueue],
+        *,
+        externals: DictData | None = None,
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from the specific input
+        datetime that mapping with the on field.
+
+            This task creation need queue to tracking release date already
+        mapped or not.
+
+        :param start_date: A start date that get from the workflow schedule.
+        :param queue: A mapping of name and list of datetime for queue.
+        :param externals: An external parameters that pass to the Loader object.
+
+        :rtype: list[WorkflowTask]
+        :return: Return the list of WorkflowTask object from the specific
+            input datetime that mapping with the on field.
+        """
+        workflow_tasks: list[WorkflowTask] = []
+        extras: DictData = externals or {}
+
+        # NOTE: Loading workflow model from the name of workflow.
+        wf: Workflow = Workflow.from_loader(self.name, externals=extras)
+        wf_queue: WorkflowQueue = queue[self.alias]
+
+        # IMPORTANT: Create the default 'on' value if it does not passing
+        #   the on field to the Schedule object.
+        ons: list[On] = self.on or wf.on.copy()
+
+        for on in ons:
+
+            # NOTE: Create CronRunner instance from the start_date param.
+            runner: CronRunner = on.generate(start_date)
+            next_running_date = runner.next
+
+            while wf_queue.check_queue(next_running_date):
+                next_running_date = runner.next
+
+            workflow_tasks.append(
+                WorkflowTask(
+                    alias=self.alias,
+                    workflow=wf,
+                    runner=runner,
+                    values=self.values,
+                ),
+            )
+
+        return workflow_tasks
+
 
 class Schedule(BaseModel):
     """Schedule Pydantic model that use to run with any scheduler package.
@@ -214,7 +267,7 @@ class Schedule(BaseModel):
         loader: Loader = Loader(name, externals=(externals or {}))
 
         # NOTE: Validate the config type match with current connection model
-        if loader.type != cls:
+        if loader.type != cls.__name__:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
 
         loader_data: DictData = copy.deepcopy(loader.data)
@@ -227,57 +280,33 @@ class Schedule(BaseModel):
     def tasks(
         self,
         start_date: datetime,
-        queue: dict[str, list[datetime]],
+        queue: dict[str, WorkflowQueue],
         *,
         externals: DictData | None = None,
-    ) -> list[WorkflowTaskData]:
-        """Return the list of WorkflowTaskData object from the specific input
-        datetime that mapping with the on field.
-
-            This task creation need queue to tracking release date already
-        mapped or not.
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from the specific input
+        datetime that mapping with the on field from workflow schedule model.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
+        :type queue: dict[str, WorkflowQueue]
         :param externals: An external parameters that pass to the Loader object.
+        :type externals: DictData | None
 
-        :rtype: list[WorkflowTaskData]
-        :return: Return the list of WorkflowTaskData object from the specific
+        :rtype: list[WorkflowTask]
+        :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
-        workflow_tasks: list[WorkflowTaskData] = []
-        extras: DictData = externals or {}
+        workflow_tasks: list[WorkflowTask] = []
 
-        for sch_wf in self.workflows:
+        for workflow in self.workflows:
 
-            # NOTE: Loading workflow model from the name of workflow.
-            wf: Workflow = Workflow.from_loader(sch_wf.name, externals=extras)
+            if workflow.alias not in queue:
+                queue[workflow.alias] = WorkflowQueue()
 
-            # NOTE: Create default list of release datetime by empty list.
-            if sch_wf.alias not in queue:
-                queue[sch_wf.alias]: list[datetime] = []
-
-            # IMPORTANT: Create the default 'on' value if it does not passing
-            #   the on field to the Schedule object.
-            ons: list[On] = sch_wf.on or wf.on.copy()
-
-            for on in ons:
-
-                # NOTE: Create CronRunner instance from the start_date param.
-                runner: CronRunner = on.generate(start_date)
-                next_running_date = runner.next
-
-                while next_running_date in queue[sch_wf.alias]:
-                    next_running_date = runner.next
-
-                workflow_tasks.append(
-                    WorkflowTaskData(
-                        alias=sch_wf.alias,
-                        workflow=wf,
-                        runner=runner,
-                        params=sch_wf.values,
-                    ),
-                )
+            workflow_tasks.extend(
+                workflow.tasks(start_date, queue=queue, externals=externals)
+            )
 
         return workflow_tasks
 
@@ -298,14 +327,6 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:  # pragma: no cov
         try:
-            # NOTE: Check the function that want to handle is method or not.
-            if inspect.ismethod(func):
-
-                @wraps(func)
-                def wrapper(self, *args, **kwargs):
-                    return func(self, *args, **kwargs)
-
-                return wrapper
 
             @wraps(func)
             def wrapper(*args, **kwargs):
@@ -322,37 +343,37 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
     return decorator
 
 
-@catch_exceptions(cancel_on_failure=True)  # pragma: no cov
-def workflow_task_release(
-    workflow_tasks: list[WorkflowTaskData],
+class ReleaseThread(TypedDict):
+    thread: Thread
+    start_date: datetime
+
+
+ReleaseThreads = dict[str, ReleaseThread]
+
+
+@catch_exceptions(cancel_on_failure=True)
+def schedule_task(
+    tasks: list[WorkflowTask],
     stop: datetime,
-    queue,
-    running,
-    threads: dict[str, Thread],
+    queue: dict[str, WorkflowQueue],
+    threads: ReleaseThreads,
+    log: type[Log],
 ) -> CancelJob | None:
     """Workflow task generator that create release pair of workflow and on to
     the threading in background.
 
         This workflow task will start every minute at ':02' second.
 
-    :param workflow_tasks:
+    :param tasks: A list of WorkflowTask object.
     :param stop: A stop datetime object that force stop running scheduler.
-    :param queue:
-    :param running:
-    :param threads:
+    :param queue: A mapping of alias name and WorkflowQueue object.
+    :param threads: A mapping of alias name and Thread object.
+    :param log: A log class that want to making log object.
+
     :rtype: CancelJob | None
     """
     current_date: datetime = datetime.now(tz=config.tz)
-
     if current_date > stop.replace(tzinfo=config.tz):
-        logger.info("[WORKFLOW]: Stop this schedule with datetime stopper.")
-        while len(threads) > 0:
-            logger.warning(
-                "[WORKFLOW]: Waiting workflow release thread that still "
-                "running in background."
-            )
-            time.sleep(15)
-            workflow_monitor(threads)
         return CancelJob
 
     # IMPORTANT:
@@ -367,91 +388,104 @@ def workflow_task_release(
     #   '00:02:00'  --> '*/2 * * * *'   --> running
     #               --> '*/35 * * * *'  --> skip
     #
-    for task in workflow_tasks:
+    for task in tasks:
+
+        q: WorkflowQueue = queue[task.alias]
+
+        # NOTE: Start adding queue and move the runner date in the WorkflowTask.
+        task.queue(stop, q, log=log)
 
         # NOTE: Get incoming datetime queue.
-        logger.debug(
-            f"[WORKFLOW]: Current queue: {task.workflow.name!r} : "
-            f"{list(queue2str(queue[task.alias]))}"
-        )
+        logger.debug(f"[WORKFLOW]: Queue: {task.alias!r} : {list(q.queue)}")
 
-        if (
-            len(queue[task.alias]) > 0
-            and task.runner.date != queue[task.alias][0]
-        ):
-            logger.debug(
-                f"[WORKFLOW]: Skip schedule "
-                f"{task.runner.date:%Y-%m-%d %H:%M:%S} "
-                f"for : {task.workflow.name!r} : {task.runner.cron}"
+        # VALIDATE: Check the queue is empty or not.
+        if not q.is_queued:
+            logger.warning(
+                f"[WORKFLOW]: Queue is empty for : {task.alias!r} : "
+                f"{task.runner.cron}"
             )
             continue
 
-        elif len(queue[task.alias]) == 0:
-            logger.warning(
-                f"[WORKFLOW]: Queue is empty for : {task.workflow.name!r} : "
-                f"{task.runner.cron}"
+        # VALIDATE: Check this task is the first release in the queue or not.
+        current_release: datetime = current_date.replace(
+            second=0, microsecond=0
+        )
+        if (first_date := q.first_queue.date) != current_release:
+            logger.debug(
+                f"[WORKFLOW]: Skip schedule "
+                f"{first_date:%Y-%m-%d %H:%M:%S} for : {task.alias!r}"
             )
             continue
 
-        # NOTE: Remove this datetime from queue.
-        queue[task.alias].pop(0)
+        # NOTE: Pop the latest release and push it to running.
+        release: WorkflowRelease = heappop(q.queue)
+        heappush(q.running, release)
 
-        # NOTE: Create thread name that able to tracking with observe schedule
-        #   job.
-        thread_name: str = (
-            f"{task.workflow.name}|{str(task.runner.cron)}|"
-            f"{task.runner.date:%Y%m%d%H%M}"
+        logger.info(
+            f"[WORKFLOW]: Start thread: '{task.alias}|"
+            f"{release.date:%Y%m%d%H%M}'"
         )
 
-        wf_thread: Thread = Thread(
+        # NOTE: Create thread name that able to tracking with observe schedule
+        #   job.
+        thread_name: str = f"{task.alias}|{release.date:%Y%m%d%H%M}"
+        thread: Thread = Thread(
             target=catch_exceptions(cancel_on_failure=True)(task.release),
-            kwargs={
-                "queue": queue,
-                "running": running,
-            },
+            kwargs={"release": release, "queue": q, "log": log},
             name=thread_name,
             daemon=True,
         )
 
-        threads[thread_name] = wf_thread
+        threads[thread_name] = {
+            "thread": thread,
+            "start_date": datetime.now(tz=config.tz),
+        }
 
-        wf_thread.start()
+        thread.start()
 
         delay()
 
-    logger.debug(f"[WORKFLOW]: {'=' * 100}")
+    logger.debug(f"[SCHEDULE]: End schedule release {'=' * 80}")
 
 
-def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
-    """Workflow schedule for monitoring long running thread from the schedule
-    control.
+def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
+    """Monitoring function that running every five minute for track long running
+    thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
-    :rtype: None
+    :type threads: ReleaseThreads
     """
     logger.debug(
         "[MONITOR]: Start checking long running workflow release task."
     )
-    snapshot_threads = list(threads.keys())
+
+    snapshot_threads: list[str] = list(threads.keys())
     for t_name in snapshot_threads:
 
+        thread_release: ReleaseThread = threads[t_name]
+
         # NOTE: remove the thread that running success.
-        if not threads[t_name].is_alive():
+        if not thread_release["thread"].is_alive():
             threads.pop(t_name)
 
 
-def workflow_control(
+def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
     externals: DictData | None = None,
+    *,
+    log: type[Log] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Workflow scheduler control.
+    """Scheduler control function that running every minute.
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: An datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
+    :param log:
+
     :rtype: list[str]
     """
+    # NOTE: Lazy import Scheduler object from the schedule package.
     try:
         from schedule import Scheduler
     except ImportError:
@@ -459,30 +493,27 @@ def workflow_control(
             "Should install schedule package before use this module."
         ) from None
 
+    log: type[Log] = log or FileLog
     scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
+    stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
 
-    # NOTE: Design workflow queue caching.
-    #   ---
-    #   {"workflow-name": [<release-datetime>, <release-datetime>, ...]}
-    #
-    wf_queue: dict[str, list[datetime]] = {}
-    thread_releases: dict[str, Thread] = {}
+    # IMPORTANT: Create main mapping of queue and thread object.
+    queue: dict[str, WorkflowQueue] = {}
+    threads: ReleaseThreads = {}
 
-    start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
+    start_date_waiting: datetime = start_date.replace(
         second=0, microsecond=0
-    )
+    ) + timedelta(minutes=1)
 
-    # NOTE: Create pair of workflow and on from schedule model.
-    workflow_tasks: list[WorkflowTaskData] = []
+    # NOTE: Start create workflow tasks from list of schedule name.
+    tasks: list[WorkflowTask] = []
     for name in schedules:
         schedule: Schedule = Schedule.from_loader(name, externals=externals)
-
-        # NOTE: Create a workflow task data instance from schedule object.
-        workflow_tasks.extend(
+        tasks.extend(
             schedule.tasks(
                 start_date_waiting,
-                queue=wf_queue,
+                queue=queue,
                 externals=externals,
             ),
         )
@@ -492,23 +523,33 @@ def workflow_control(
         scheduler.every(1)
         .minutes.at(":02")
         .do(
-            workflow_task_release,
-            workflow_tasks=workflow_tasks,
-            stop=(stop or (start_date + config.stop_boundary_delta)),
-            queue=wf_queue,
-            threads=thread_releases,
+            schedule_task,
+            tasks=tasks,
+            stop=stop_date,
+            queue=queue,
+            threads=threads,
+            log=log,
         )
         .tag("control")
     )
 
     # NOTE: Checking zombie task with schedule job will start every 5 minute.
-    scheduler.every(5).minutes.at(":10").do(
-        workflow_monitor,
-        threads=thread_releases,
-    ).tag("monitor")
+    (
+        scheduler.every(5)
+        .minutes.at(":10")
+        .do(
+            monitor,
+            threads=threads,
+        )
+        .tag("monitor")
+    )
 
     # NOTE: Start running schedule
-    logger.info(f"[WORKFLOW]: Start schedule: {schedules}")
+    logger.info(
+        f"[SCHEDULE]: Schedule: {schedules} with stopper: "
+        f"{stop_date:%Y-%m-%d %H:%M:%S}"
+    )
+
     while True:
         scheduler.run_pending()
         time.sleep(1)
@@ -516,29 +557,35 @@ def workflow_control(
         # NOTE: Break the scheduler when the control job does not exists.
         if not scheduler.get_jobs("control"):
             scheduler.clear("monitor")
-            logger.warning(
-                f"[WORKFLOW]: Workflow release thread: {thread_releases}"
-            )
-            logger.warning("[WORKFLOW]: Does not have any schedule jobs !!!")
+
+            while len(threads) > 0:
+                logger.warning(
+                    "[SCHEDULE]: Waiting schedule release thread that still "
+                    "running in background."
+                )
+                delay(15)
+                monitor(threads)
+
             break
 
     logger.warning(
-        f"Queue: {[list(queue2str(wf_queue[wf])) for wf in wf_queue]}"
+        f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
     )
     return schedules
 
 
-def workflow_runner(
+def schedule_runner(
     stop: datetime | None = None,
     externals: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Workflow application that running multiprocessing schedule with chunk of
-    workflows that exists in config path.
+    """Schedule runner function for start submit the ``schedule_control`` func
+    in multiprocessing pool with chunk of schedule config that exists in config
+    path by ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS``.
 
     :param stop: A stop datetime object that force stop running scheduler.
-    :param excluded:
     :param externals:
+    :param excluded: A list of schedule name that want to excluded from finding.
 
     :rtype: list[str]
 
@@ -549,24 +596,21 @@ def workflow_runner(
 
         The current workflow logic that split to process will be below diagram:
 
-        PIPELINES ==> process 01 ==> schedule --> thread of release
-                                                  workflow task 01 01
-                                              --> thread of release
-                                                  workflow task 01 02
-                  ==> process 02 ==> schedule --> thread of release
-                                                  workflow task 02 01
-                                              --> thread of release
-                                                  workflow task 02 02
-                  ==> ...
+        MAIN ==> process 01 ==> schedule --> thread of release task 01 01
+                                        --> thread of release task 01 02
+                            ==> schedule --> thread of release task 02 01
+                                        --> thread of release task 02 02
+            ==> process 02
     """
-    excluded: list[str] = excluded or []
+    results: list[str] = []
 
     with ProcessPoolExecutor(
         max_workers=config.max_schedule_process,
     ) as executor:
+
         futures: list[Future] = [
             executor.submit(
-                workflow_control,
+                schedule_control,
                 schedules=[load[0] for load in loader],
                 stop=stop,
                 externals=(externals or {}),
@@ -577,10 +621,13 @@ def workflow_runner(
             )
         ]
 
-        results: list[str] = []
         for future in as_completed(futures):
+
+            # NOTE: Raise error when it has any error from schedule_control.
             if err := future.exception():
                 logger.error(str(err))
                 raise WorkflowException(str(err)) from err
+
             results.extend(future.result(timeout=1))
-        return results
+
+    return results

ddeutil/workflow/stage.py

@@ -346,6 +346,11 @@ class EmptyStage(BaseStage):
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
+            if self.sleep > 30:
+                logger.info(
+                    f"({cut_id(run_id)}) [STAGE]: ... sleep "
+                    f"({self.sleep} seconds)"
+                )
             time.sleep(self.sleep)
         return Result(status=0, context={}, run_id=run_id)
 

ddeutil/workflow/utils.py

@@ -74,6 +74,12 @@ def get_diff_sec(
     )
 
 
+def wait_a_minute(now: datetime, second: float = 2) -> None:  # pragma: no cov
+    """Wait with sleep to the next minute with an offset second value."""
+    future = now.replace(second=0, microsecond=0) + timedelta(minutes=1)
+    time.sleep((future - now).total_seconds() + second)
+
+
 def delay(second: float = 0) -> None:  # pragma: no cov
     """Delay time that use time.sleep with random second value between
     0.00 - 0.99 seconds.
@@ -326,7 +332,7 @@ def get_args_from_filter(
 
     if func_name not in filters:
         raise UtilException(
-            f"The post-filter: {func_name} does not support yet."
+            f"The post-filter: {func_name!r} does not support yet."
         )
 
     if isinstance((f_func := filters[func_name]), list) and (args or kwargs):
@@ -577,10 +583,6 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         yield chain((first_el,), chunk_it)
 
 
-def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
-    return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
-
-
 def cut_id(run_id: str, *, num: int = 6):
     """Cutting running ID with length.