ddeutil-workflow 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 """
-The main schedule running is ``workflow_runner`` function that trigger the
+The main schedule running is ``schedule_runner`` function that trigger the
 multiprocess of ``workflow_control`` function for listing schedules on the
 config by ``Loader.finds(Schedule)``.
 
@@ -21,7 +21,6 @@ value with the on field.
 from __future__ import annotations
 
 import copy
-import inspect
 import logging
 import time
 from concurrent.futures import (
@@ -31,6 +30,7 @@ from concurrent.futures import (
 )
 from datetime import datetime, timedelta
 from functools import wraps
+from heapq import heappop
 from textwrap import dedent
 from threading import Thread
 from typing import Callable, Optional
@@ -51,52 +51,55 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, config, get_logger
+from .conf import FileLog, Loader, Log, config, get_logger
+from .cron import On
 from .exceptions import WorkflowException
-from .on import On
 from .utils import (
     batch,
     delay,
-    queue2str,
 )
-from .workflow import Workflow, WorkflowTaskData
+from .workflow import Workflow, WorkflowQueue, WorkflowRelease, WorkflowTask
 
 P = ParamSpec("P")
 logger = get_logger("ddeutil.workflow")
 
-# NOTE: Adjust logging level on the schedule package.
+# NOTE: Adjust logging level on the `schedule` package.
 logging.getLogger("schedule").setLevel(logging.INFO)
 
 
 __all__: TupleStr = (
     "Schedule",
-    "ScheduleWorkflow",
-    "workflow_task_release",
-    "workflow_monitor",
-    "workflow_control",
-    "workflow_runner",
+    "WorkflowSchedule",
+    "schedule_task",
+    "monitor",
+    "schedule_control",
+    "schedule_runner",
 )
 
 
-class ScheduleWorkflow(BaseModel):
-    """Schedule Workflow Pydantic model that use to keep workflow model for the
-    Schedule model. it should not use Workflow model directly because on the
+class WorkflowSchedule(BaseModel):
+    """Workflow Schedule Pydantic model that use to keep workflow model for
+    the Schedule model. it should not use Workflow model directly because on the
     schedule config it can adjust crontab value that different from the Workflow
     model.
     """
 
     alias: Optional[str] = Field(
         default=None,
-        description="An alias name of workflow.",
+        description="An alias name of workflow that use for schedule model.",
     )
     name: str = Field(description="A workflow name.")
     on: list[On] = Field(
         default_factory=list,
-        description="An override On instance value.",
+        description="An override the list of On object values.",
     )
-    params: DictData = Field(
+    values: DictData = Field(
         default_factory=dict,
-        description="A parameters that want to use in workflow execution.",
+        description=(
+            "A value that want to pass to the workflow parameters when "
+            "calling release method."
+        ),
+        alias="params",
     )
 
     @model_validator(mode="before")
@@ -105,10 +108,13 @@ class ScheduleWorkflow(BaseModel):
 
         :rtype: DictData
         """
-        values["name"] = values["name"].replace(" ", "_")
+        # VALIDATE: Prepare a workflow name that should not include space.
+        if name := values.get("name"):
+            values["name"] = name.replace(" ", "_")
 
+        # VALIDATE: Add default the alias field with the name.
         if not values.get("alias"):
-            values["alias"] = values["name"]
+            values["alias"] = values.get("name")
 
         cls.__bypass_on(values)
         return values
@@ -135,6 +141,7 @@ class ScheduleWorkflow(BaseModel):
                 Loader(n, externals={}).data if isinstance(n, str) else n
                 for n in on
             ]
+
         return data
 
     @field_validator("on", mode="after")
@@ -150,19 +157,72 @@ class ScheduleWorkflow(BaseModel):
                 "The on fields should not contain duplicate on value."
             )
 
-        # WARNING:
-        # if '* * * * *' in set_ons and len(set_ons) > 1:
-        #     raise ValueError(
-        #         "If it has every minute cronjob on value, it should has only "
-        #         "one value in the on field."
-        #     )
+        if len(set_ons) > config.max_on_per_workflow:
+            raise ValueError(
+                f"The number of the on should not more than "
+                f"{config.max_on_per_workflow} crontab."
+            )
+
         return value
 
+    def tasks(
+        self,
+        start_date: datetime,
+        queue: dict[str, WorkflowQueue],
+        *,
+        externals: DictData | None = None,
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from the specific input
+        datetime that mapping with the on field.
+
+            This task creation need queue to tracking release date already
+        mapped or not.
+
+        :param start_date: A start date that get from the workflow schedule.
+        :param queue: A mapping of name and list of datetime for queue.
+        :param externals: An external parameters that pass to the Loader object.
+
+        :rtype: list[WorkflowTask]
+        :return: Return the list of WorkflowTask object from the specific
+            input datetime that mapping with the on field.
+        """
+        workflow_tasks: list[WorkflowTask] = []
+        extras: DictData = externals or {}
+
+        # NOTE: Loading workflow model from the name of workflow.
+        wf: Workflow = Workflow.from_loader(self.name, externals=extras)
+        wf_queue: WorkflowQueue = queue[self.alias]
+
+        # IMPORTANT: Create the default 'on' value if it does not passing
+        #   the on field to the Schedule object.
+        ons: list[On] = self.on or wf.on.copy()
+
+        for on in ons:
+
+            # NOTE: Create CronRunner instance from the start_date param.
+            runner: CronRunner = on.generate(start_date)
+            next_running_date = runner.next
+
+            while wf_queue.check_queue(next_running_date):
+                next_running_date = runner.next
+
+            workflow_tasks.append(
+                WorkflowTask(
+                    alias=self.alias,
+                    workflow=wf,
+                    runner=runner,
+                    values=self.values,
+                ),
+            )
+
+        return workflow_tasks
+
 
 class Schedule(BaseModel):
-    """Schedule Pydantic Model that use to run with scheduler package. It does
-    not equal the on value in Workflow model but it use same logic to running
-    release date with crontab interval.
+    """Schedule Pydantic model that use to run with any scheduler package.
+
+        It does not equal the on value in Workflow model but it use same logic
+    to running release date with crontab interval.
     """
 
     desc: Optional[str] = Field(
@@ -171,9 +231,9 @@ class Schedule(BaseModel):
             "A schedule description that can be string of markdown content."
         ),
     )
-    workflows: list[ScheduleWorkflow] = Field(
+    workflows: list[WorkflowSchedule] = Field(
         default_factory=list,
-        description="A list of ScheduleWorkflow models.",
+        description="A list of WorkflowSchedule models.",
     )
 
     @field_validator("desc", mode="after")
@@ -181,6 +241,7 @@ class Schedule(BaseModel):
         """Prepare description string that was created on a template.
 
         :param value: A description string value that want to dedent.
+
         :rtype: str
         """
         return dedent(value)
@@ -217,55 +278,33 @@ class Schedule(BaseModel):
     def tasks(
         self,
         start_date: datetime,
-        queue: dict[str, list[datetime]],
+        queue: dict[str, WorkflowQueue],
         *,
         externals: DictData | None = None,
-    ) -> list[WorkflowTaskData]:
-        """Return the list of WorkflowTaskData object from the specific input
-        datetime that mapping with the on field.
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from the specific input
+        datetime that mapping with the on field from workflow schedule model.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
+        :type queue: dict[str, WorkflowQueue]
         :param externals: An external parameters that pass to the Loader object.
+        :type externals: DictData | None
 
-        :rtype: list[WorkflowTaskData]
-        :return: Return the list of WorkflowTaskData object from the specific
+        :rtype: list[WorkflowTask]
+        :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
+        workflow_tasks: list[WorkflowTask] = []
 
-        # NOTE: Create pair of workflow and on.
-        workflow_tasks: list[WorkflowTaskData] = []
-        extras: DictData = externals or {}
-
-        for sch_wf in self.workflows:
-
-            wf: Workflow = Workflow.from_loader(sch_wf.name, externals=extras)
-
-            # NOTE: Create default list of release datetime.
-            if sch_wf.alias not in queue:
-                queue[sch_wf.alias]: list[datetime] = []
-
-            # IMPORTANT: Create the default 'on' value if it does not passing
-            #   the on field to the Schedule object.
-            ons: list[On] = wf.on.copy() if len(sch_wf.on) == 0 else sch_wf.on
-
-            for on in ons:
-
-                # NOTE: Create CronRunner instance from the start_date param.
-                runner: CronRunner = on.generate(start_date)
-                next_running_date = runner.next
+        for workflow in self.workflows:
 
-                while next_running_date in queue[sch_wf.alias]:
-                    next_running_date = runner.next
+            if workflow.alias not in queue:
+                queue[workflow.alias] = WorkflowQueue()
 
-                workflow_tasks.append(
-                    WorkflowTaskData(
-                        alias=sch_wf.alias,
-                        workflow=wf,
-                        runner=runner,
-                        params=sch_wf.params,
-                    ),
-                )
+            workflow_tasks.extend(
+                workflow.tasks(start_date, queue=queue, externals=externals)
+            )
 
         return workflow_tasks
 
@@ -286,14 +325,6 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:  # pragma: no cov
         try:
-            # NOTE: Check the function that want to handle is method or not.
-            if inspect.ismethod(func):
-
-                @wraps(func)
-                def wrapper(self, *args, **kwargs):
-                    return func(self, *args, **kwargs)
-
-                return wrapper
 
             @wraps(func)
             def wrapper(*args, **kwargs):
@@ -311,36 +342,28 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
 
 @catch_exceptions(cancel_on_failure=True)  # pragma: no cov
-def workflow_task_release(
-    workflow_tasks: list[WorkflowTaskData],
+def schedule_task(
+    tasks: list[WorkflowTask],
     stop: datetime,
-    queue,
-    running,
+    queue: dict[str, WorkflowQueue],
     threads: dict[str, Thread],
+    log: type[Log],
 ) -> CancelJob | None:
     """Workflow task generator that create release pair of workflow and on to
     the threading in background.
 
         This workflow task will start every minute at ':02' second.
 
-    :param workflow_tasks:
+    :param tasks: A list of WorkflowTask object.
     :param stop: A stop datetime object that force stop running scheduler.
-    :param queue:
-    :param running:
-    :param threads:
+    :param queue: A mapping of alias name and WorkflowQueue object.
+    :param threads: A mapping of alias name and Thread object.
+    :param log: A log class that want to making log object.
+
     :rtype: CancelJob | None
     """
     current_date: datetime = datetime.now(tz=config.tz)
-
     if current_date > stop.replace(tzinfo=config.tz):
-        logger.info("[WORKFLOW]: Stop this schedule with datetime stopper.")
-        while len(threads) > 0:
-            logger.warning(
-                "[WORKFLOW]: Waiting workflow release thread that still "
-                "running in background."
-            )
-            time.sleep(15)
-            workflow_monitor(threads)
         return CancelJob
 
     # IMPORTANT:
@@ -355,48 +378,53 @@ def workflow_task_release(
     #   '00:02:00'  --> '*/2 * * * *'   --> running
     #               --> '*/35 * * * *'  --> skip
     #
-    for task in workflow_tasks:
+    for task in tasks:
+
+        q: WorkflowQueue = queue[task.alias]
+
+        # NOTE: Start adding queue and move the runner date in the WorkflowTask.
+        task.queue(stop, q, log=log)
 
         # NOTE: Get incoming datetime queue.
-        logger.debug(
-            f"[WORKFLOW]: Current queue: {task.workflow.name!r} : "
-            f"{list(queue2str(queue[task.alias]))}"
-        )
+        logger.debug(f"[WORKFLOW]: Queue: {task.alias!r} : {list(q.queue)}")
 
-        if (
-            len(queue[task.alias]) > 0
-            and task.runner.date != queue[task.alias][0]
-        ):
-            logger.debug(
-                f"[WORKFLOW]: Skip schedule "
-                f"{task.runner.date:%Y-%m-%d %H:%M:%S} "
-                f"for : {task.workflow.name!r} : {task.runner.cron}"
+        # VALIDATE: Check the queue is empty or not.
+        if not q.is_queued:
+            logger.warning(
+                f"[WORKFLOW]: Queue is empty for : {task.alias!r} : "
+                f"{task.runner.cron}"
             )
             continue
 
-        elif len(queue[task.alias]) == 0:
-            logger.warning(
-                f"[WORKFLOW]: Queue is empty for : {task.workflow.name!r} : "
-                f"{task.runner.cron}"
+        # VALIDATE: Check this task is the first release in the queue or not.
+        current_date: datetime = current_date.replace(second=0, microsecond=0)
+        if (first_date := q.first_queue.date) != current_date:
+            logger.debug(
+                f"[WORKFLOW]: Skip schedule "
+                f"{first_date:%Y-%m-%d %H:%M:%S} "
+                f"for : {task.alias!r} : {task.runner.cron}"
             )
             continue
 
-        # NOTE: Remove this datetime from queue.
-        queue[task.alias].pop(0)
+        # NOTE: Pop the latest release and push it to running.
+        release: WorkflowRelease = heappop(q.queue)
+        q.push_running(release)
+
+        logger.info(
+            f"[WORKFLOW]: Start thread: '{task.alias}|{str(task.runner.cron)}|"
+            f"{release.date:%Y%m%d%H%M}'"
+        )
 
         # NOTE: Create thread name that able to tracking with observe schedule
         #   job.
         thread_name: str = (
-            f"{task.workflow.name}|{str(task.runner.cron)}|"
-            f"{task.runner.date:%Y%m%d%H%M}"
+            f"{task.alias}|{str(task.runner.cron)}|"
+            f"{release.date:%Y%m%d%H%M}"
         )
 
         wf_thread: Thread = Thread(
             target=catch_exceptions(cancel_on_failure=True)(task.release),
-            kwargs={
-                "queue": queue,
-                "running": running,
-            },
+            kwargs={"release": release, "queue": q, "log": log},
             name=thread_name,
             daemon=True,
         )
@@ -407,20 +435,21 @@ def workflow_task_release(
 
         delay()
 
-    logger.debug(f"[WORKFLOW]: {'=' * 100}")
+    logger.debug(f"[SCHEDULE]: End schedule release {'=' * 80}")
 
 
-def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
-    """Workflow schedule for monitoring long running thread from the schedule
-    control.
+def monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
+    """Monitoring function that running every five minute for track long running
+    thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
-    :rtype: None
+    :type threads: dict[str, Thread]
     """
     logger.debug(
         "[MONITOR]: Start checking long running workflow release task."
     )
-    snapshot_threads = list(threads.keys())
+
+    snapshot_threads: list[str] = list(threads.keys())
     for t_name in snapshot_threads:
 
         # NOTE: remove the thread that running success.
@@ -428,18 +457,23 @@ def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
             threads.pop(t_name)
 
 
-def workflow_control(
+def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
     externals: DictData | None = None,
+    *,
+    log: type[Log] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Workflow scheduler control.
+    """Scheduler control function that running every minute.
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: An datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
+    :param log:
+
     :rtype: list[str]
     """
+    # NOTE: Lazy import Scheduler object from the schedule package.
     try:
         from schedule import Scheduler
     except ImportError:
@@ -447,30 +481,27 @@ def workflow_control(
             "Should install schedule package before use this module."
         ) from None
 
+    log: type[Log] = log or FileLog
     scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
+    stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
 
-    # NOTE: Design workflow queue caching.
-    #   ---
-    #   {"workflow-name": [<release-datetime>, <release-datetime>, ...]}
-    #
-    wf_queue: dict[str, list[datetime]] = {}
-    thread_releases: dict[str, Thread] = {}
+    # IMPORTANT: Create main mapping of queue and thread object.
+    queue: dict[str, WorkflowQueue] = {}
+    threads: dict[str, Thread] = {}
 
-    start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
+    start_date_waiting: datetime = start_date.replace(
         second=0, microsecond=0
-    )
+    ) + timedelta(minutes=1)
 
-    # NOTE: Create pair of workflow and on from schedule model.
-    workflow_tasks: list[WorkflowTaskData] = []
+    # NOTE: Start create workflow tasks from list of schedule name.
+    tasks: list[WorkflowTask] = []
     for name in schedules:
         schedule: Schedule = Schedule.from_loader(name, externals=externals)
-
-        # NOTE: Create a workflow task data instance from schedule object.
-        workflow_tasks.extend(
+        tasks.extend(
             schedule.tasks(
                 start_date_waiting,
-                queue=wf_queue,
+                queue=queue,
                 externals=externals,
             ),
         )
@@ -480,23 +511,33 @@ def workflow_control(
         scheduler.every(1)
         .minutes.at(":02")
         .do(
-            workflow_task_release,
-            workflow_tasks=workflow_tasks,
-            stop=(stop or (start_date + config.stop_boundary_delta)),
-            queue=wf_queue,
-            threads=thread_releases,
+            schedule_task,
+            tasks=tasks,
+            stop=stop_date,
+            queue=queue,
+            threads=threads,
+            log=log,
         )
         .tag("control")
     )
 
     # NOTE: Checking zombie task with schedule job will start every 5 minute.
-    scheduler.every(5).minutes.at(":10").do(
-        workflow_monitor,
-        threads=thread_releases,
-    ).tag("monitor")
+    (
+        scheduler.every(5)
+        .minutes.at(":10")
+        .do(
+            monitor,
+            threads=threads,
+        )
+        .tag("monitor")
+    )
 
     # NOTE: Start running schedule
-    logger.info(f"[WORKFLOW]: Start schedule: {schedules}")
+    logger.info(
+        f"[SCHEDULE]: Schedule: {schedules} with stopper: "
+        f"{stop_date:%Y-%m-%d %H:%M:%S}"
+    )
+
     while True:
         scheduler.run_pending()
         time.sleep(1)
@@ -504,29 +545,35 @@ def workflow_control(
         # NOTE: Break the scheduler when the control job does not exists.
         if not scheduler.get_jobs("control"):
             scheduler.clear("monitor")
-            logger.warning(
-                f"[WORKFLOW]: Workflow release thread: {thread_releases}"
-            )
-            logger.warning("[WORKFLOW]: Does not have any schedule jobs !!!")
+
+            while len(threads) > 0:
+                logger.warning(
+                    "[SCHEDULE]: Waiting schedule release thread that still "
+                    "running in background."
+                )
+                delay(15)
+                monitor(threads)
+
             break
 
     logger.warning(
-        f"Queue: {[list(queue2str(wf_queue[wf])) for wf in wf_queue]}"
+        f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
     )
     return schedules
 
 
-def workflow_runner(
+def schedule_runner(
     stop: datetime | None = None,
     externals: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Workflow application that running multiprocessing schedule with chunk of
-    workflows that exists in config path.
+    """Schedule runner function for start submit the ``schedule_control`` func
+    in multiprocessing pool with chunk of schedule config that exists in config
+    path by ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS``.
 
     :param stop: A stop datetime object that force stop running scheduler.
-    :param excluded:
     :param externals:
+    :param excluded: A list of schedule name that want to excluded from finding.
 
     :rtype: list[str]
 
@@ -537,24 +584,21 @@ def workflow_runner(
 
         The current workflow logic that split to process will be below diagram:
 
-        PIPELINES ==> process 01 ==> schedule --> thread of release
-                                                  workflow task 01 01
-                                              --> thread of release
-                                                  workflow task 01 02
-                  ==> process 02 ==> schedule --> thread of release
-                                                  workflow task 02 01
-                                              --> thread of release
-                                                  workflow task 02 02
-                  ==> ...
+        MAIN ==> process 01 ==> schedule --> thread of release task 01 01
+                                        --> thread of release task 01 02
+                            ==> schedule --> thread of release task 02 01
+                                        --> thread of release task 02 02
+            ==> process 02
     """
-    excluded: list[str] = excluded or []
+    results: list[str] = []
 
     with ProcessPoolExecutor(
         max_workers=config.max_schedule_process,
     ) as executor:
+
         futures: list[Future] = [
             executor.submit(
-                workflow_control,
+                schedule_control,
                 schedules=[load[0] for load in loader],
                 stop=stop,
                 externals=(externals or {}),
@@ -565,10 +609,13 @@ def workflow_runner(
             )
         ]
 
-        results: list[str] = []
         for future in as_completed(futures):
+
+            # NOTE: Raise error when it has any error from schedule_control.
             if err := future.exception():
                 logger.error(str(err))
                 raise WorkflowException(str(err)) from err
+
             results.extend(future.result(timeout=1))
-        return results
+
+    return results