ddeutil-workflow 0.0.23__py3-none-any.whl → 0.0.24__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.23"
+__version__: str = "0.0.24"

ddeutil/workflow/__cron.py

@@ -736,6 +736,12 @@ class CronRunner:
         self.is_year: bool = isinstance(cron, CronJobYear)
         self.reset_flag: bool = True
 
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(CronJob('{self.cron}'), "
+            f"{self.date:%Y-%m-%d %H:%M:%S}, tz='{self.tz}')"
+        )
+
     def reset(self) -> None:
         """Resets the iterator to start time."""
         self.date: datetime = self.__start_date

ddeutil/workflow/__init__.py

@@ -3,11 +3,17 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+from .__cron import CronRunner
 from .conf import (
     Config,
     FileLog,
     Loader,
 )
+from .cron import (
+    On,
+    YearOn,
+    interval2crontab,
+)
 from .exceptions import (
     JobException,
     ParamValueException,
@@ -19,11 +25,6 @@ from .job import (
     Job,
     Strategy,
 )
-from .on import (
-    On,
-    YearOn,
-    interval2crontab,
-)
 from .params import (
     ChoiceParam,
     DatetimeParam,
@@ -35,6 +36,7 @@ from .result import Result
 from .scheduler import (
     Schedule,
     WorkflowSchedule,
+    schedule_runner,
 )
 from .stage import (
     BashStage,
@@ -68,11 +70,10 @@ from .utils import (
     map_post_filter,
     not_in_template,
     param2template,
-    queue2str,
     str2template,
     tag,
 )
 from .workflow import (
     Workflow,
-    WorkflowTaskData,
+    WorkflowTask,
 )

ddeutil/workflow/api.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel
 from .__about__ import __version__
 from .conf import config, get_logger
 from .repeat import repeat_at, repeat_every
-from .workflow import WorkflowTaskData
+from .workflow import WorkflowTask
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -34,7 +34,7 @@ class State(TypedDict):
     upper_result: dict[str, str]
     scheduler: list[str]
     workflow_threads: dict[str, Thread]
-    workflow_tasks: list[WorkflowTaskData]
+    workflow_tasks: list[WorkflowTask]
     workflow_queue: dict[str, list[datetime]]
     workflow_running: dict[str, list[datetime]]
 

ddeutil/workflow/cli.py

@@ -73,10 +73,10 @@ def schedule(
     if stop:
         stop: datetime = stop.astimezone(tz=config.tz)
 
-    from .scheduler import workflow_runner
+    from .scheduler import schedule_runner
 
     # NOTE: Start running workflow scheduler application.
-    workflow_rs: list[str] = workflow_runner(
+    workflow_rs: list[str] = schedule_runner(
         stop=stop, excluded=excluded, externals=json.loads(externals)
     )
     logger.info(f"Application run success: {workflow_rs}")

ddeutil/workflow/conf.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData
+from .__types import DictData, TupleStr
 
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 AnyModelType = type[AnyModel]
@@ -32,6 +32,19 @@ load_dotenv()
 
 env = os.getenv
 
+__all__: TupleStr = (
+    "get_logger",
+    "Config",
+    "SimLoad",
+    "Loader",
+    "get_type",
+    "config",
+    "logger",
+    "FileLog",
+    "SQLiteLog",
+    "Log",
+)
+
 
 @lru_cache
 def get_logger(name: str):
@@ -107,7 +120,10 @@ class Config:
         os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
     )
     max_on_per_workflow: int = int(
-        env("WORKFLOW_CORE_MAX_ON_PER_WORKFLOW", "5")
+        env("WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW", "5")
+    )
+    max_queue_complete_hist: int = int(
+        os.getenv("WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST", "16")
     )
 
     # NOTE: Schedule App

ddeutil/workflow/scheduler.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 """
-The main schedule running is ``workflow_runner`` function that trigger the
+The main schedule running is ``schedule_runner`` function that trigger the
 multiprocess of ``workflow_control`` function for listing schedules on the
 config by ``Loader.finds(Schedule)``.
 
@@ -21,7 +21,6 @@ value with the on field.
 from __future__ import annotations
 
 import copy
-import inspect
 import logging
 import time
 from concurrent.futures import (
@@ -31,6 +30,7 @@ from concurrent.futures import (
 )
 from datetime import datetime, timedelta
 from functools import wraps
+from heapq import heappop
 from textwrap import dedent
 from threading import Thread
 from typing import Callable, Optional
@@ -51,30 +51,29 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, config, get_logger
+from .conf import FileLog, Loader, Log, config, get_logger
+from .cron import On
 from .exceptions import WorkflowException
-from .on import On
 from .utils import (
     batch,
     delay,
-    queue2str,
 )
-from .workflow import Workflow, WorkflowTaskData
+from .workflow import Workflow, WorkflowQueue, WorkflowRelease, WorkflowTask
 
 P = ParamSpec("P")
 logger = get_logger("ddeutil.workflow")
 
-# NOTE: Adjust logging level on the schedule package.
+# NOTE: Adjust logging level on the `schedule` package.
 logging.getLogger("schedule").setLevel(logging.INFO)
 
 
 __all__: TupleStr = (
     "Schedule",
     "WorkflowSchedule",
-    "workflow_task_release",
-    "workflow_monitor",
-    "workflow_control",
-    "workflow_runner",
+    "schedule_task",
+    "monitor",
+    "schedule_control",
+    "schedule_runner",
 )
 
 
@@ -166,6 +165,58 @@ class WorkflowSchedule(BaseModel):
 
         return value
 
+    def tasks(
+        self,
+        start_date: datetime,
+        queue: dict[str, WorkflowQueue],
+        *,
+        externals: DictData | None = None,
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from the specific input
+        datetime that mapping with the on field.
+
+            This task creation need queue to tracking release date already
+        mapped or not.
+
+        :param start_date: A start date that get from the workflow schedule.
+        :param queue: A mapping of name and list of datetime for queue.
+        :param externals: An external parameters that pass to the Loader object.
+
+        :rtype: list[WorkflowTask]
+        :return: Return the list of WorkflowTask object from the specific
+            input datetime that mapping with the on field.
+        """
+        workflow_tasks: list[WorkflowTask] = []
+        extras: DictData = externals or {}
+
+        # NOTE: Loading workflow model from the name of workflow.
+        wf: Workflow = Workflow.from_loader(self.name, externals=extras)
+        wf_queue: WorkflowQueue = queue[self.alias]
+
+        # IMPORTANT: Create the default 'on' value if it does not passing
+        #   the on field to the Schedule object.
+        ons: list[On] = self.on or wf.on.copy()
+
+        for on in ons:
+
+            # NOTE: Create CronRunner instance from the start_date param.
+            runner: CronRunner = on.generate(start_date)
+            next_running_date = runner.next
+
+            while wf_queue.check_queue(next_running_date):
+                next_running_date = runner.next
+
+            workflow_tasks.append(
+                WorkflowTask(
+                    alias=self.alias,
+                    workflow=wf,
+                    runner=runner,
+                    values=self.values,
+                ),
+            )
+
+        return workflow_tasks
+
 
 class Schedule(BaseModel):
     """Schedule Pydantic model that use to run with any scheduler package.
@@ -227,57 +278,33 @@ class Schedule(BaseModel):
     def tasks(
         self,
         start_date: datetime,
-        queue: dict[str, list[datetime]],
+        queue: dict[str, WorkflowQueue],
         *,
         externals: DictData | None = None,
-    ) -> list[WorkflowTaskData]:
-        """Return the list of WorkflowTaskData object from the specific input
-        datetime that mapping with the on field.
-
-            This task creation need queue to tracking release date already
-        mapped or not.
+    ) -> list[WorkflowTask]:
+        """Return the list of WorkflowTask object from the specific input
+        datetime that mapping with the on field from workflow schedule model.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
+        :type queue: dict[str, WorkflowQueue]
         :param externals: An external parameters that pass to the Loader object.
+        :type externals: DictData | None
 
-        :rtype: list[WorkflowTaskData]
-        :return: Return the list of WorkflowTaskData object from the specific
+        :rtype: list[WorkflowTask]
+        :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
-        workflow_tasks: list[WorkflowTaskData] = []
-        extras: DictData = externals or {}
-
-        for sch_wf in self.workflows:
-
-            # NOTE: Loading workflow model from the name of workflow.
-            wf: Workflow = Workflow.from_loader(sch_wf.name, externals=extras)
+        workflow_tasks: list[WorkflowTask] = []
 
-            # NOTE: Create default list of release datetime by empty list.
-            if sch_wf.alias not in queue:
-                queue[sch_wf.alias]: list[datetime] = []
+        for workflow in self.workflows:
 
-            # IMPORTANT: Create the default 'on' value if it does not passing
-            #   the on field to the Schedule object.
-            ons: list[On] = sch_wf.on or wf.on.copy()
+            if workflow.alias not in queue:
+                queue[workflow.alias] = WorkflowQueue()
 
-            for on in ons:
-
-                # NOTE: Create CronRunner instance from the start_date param.
-                runner: CronRunner = on.generate(start_date)
-                next_running_date = runner.next
-
-                while next_running_date in queue[sch_wf.alias]:
-                    next_running_date = runner.next
-
-                workflow_tasks.append(
-                    WorkflowTaskData(
-                        alias=sch_wf.alias,
-                        workflow=wf,
-                        runner=runner,
-                        params=sch_wf.values,
-                    ),
-                )
+            workflow_tasks.extend(
+                workflow.tasks(start_date, queue=queue, externals=externals)
+            )
 
         return workflow_tasks
 
@@ -298,14 +325,6 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:  # pragma: no cov
         try:
-            # NOTE: Check the function that want to handle is method or not.
-            if inspect.ismethod(func):
-
-                @wraps(func)
-                def wrapper(self, *args, **kwargs):
-                    return func(self, *args, **kwargs)
-
-                return wrapper
 
             @wraps(func)
             def wrapper(*args, **kwargs):
@@ -323,36 +342,28 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
 
 @catch_exceptions(cancel_on_failure=True)  # pragma: no cov
-def workflow_task_release(
-    workflow_tasks: list[WorkflowTaskData],
+def schedule_task(
+    tasks: list[WorkflowTask],
     stop: datetime,
-    queue,
-    running,
+    queue: dict[str, WorkflowQueue],
     threads: dict[str, Thread],
+    log: type[Log],
 ) -> CancelJob | None:
     """Workflow task generator that create release pair of workflow and on to
     the threading in background.
 
         This workflow task will start every minute at ':02' second.
 
-    :param workflow_tasks:
+    :param tasks: A list of WorkflowTask object.
     :param stop: A stop datetime object that force stop running scheduler.
-    :param queue:
-    :param running:
-    :param threads:
+    :param queue: A mapping of alias name and WorkflowQueue object.
+    :param threads: A mapping of alias name and Thread object.
+    :param log: A log class that want to making log object.
+
     :rtype: CancelJob | None
     """
     current_date: datetime = datetime.now(tz=config.tz)
-
     if current_date > stop.replace(tzinfo=config.tz):
-        logger.info("[WORKFLOW]: Stop this schedule with datetime stopper.")
-        while len(threads) > 0:
-            logger.warning(
-                "[WORKFLOW]: Waiting workflow release thread that still "
-                "running in background."
-            )
-            time.sleep(15)
-            workflow_monitor(threads)
         return CancelJob
 
     # IMPORTANT:
@@ -367,48 +378,53 @@ def workflow_task_release(
     #   '00:02:00'  --> '*/2 * * * *'   --> running
     #               --> '*/35 * * * *'  --> skip
     #
-    for task in workflow_tasks:
+    for task in tasks:
+
+        q: WorkflowQueue = queue[task.alias]
+
+        # NOTE: Start adding queue and move the runner date in the WorkflowTask.
+        task.queue(stop, q, log=log)
 
         # NOTE: Get incoming datetime queue.
-        logger.debug(
-            f"[WORKFLOW]: Current queue: {task.workflow.name!r} : "
-            f"{list(queue2str(queue[task.alias]))}"
-        )
+        logger.debug(f"[WORKFLOW]: Queue: {task.alias!r} : {list(q.queue)}")
 
-        if (
-            len(queue[task.alias]) > 0
-            and task.runner.date != queue[task.alias][0]
-        ):
-            logger.debug(
-                f"[WORKFLOW]: Skip schedule "
-                f"{task.runner.date:%Y-%m-%d %H:%M:%S} "
-                f"for : {task.workflow.name!r} : {task.runner.cron}"
+        # VALIDATE: Check the queue is empty or not.
+        if not q.is_queued:
+            logger.warning(
+                f"[WORKFLOW]: Queue is empty for : {task.alias!r} : "
+                f"{task.runner.cron}"
             )
             continue
 
-        elif len(queue[task.alias]) == 0:
-            logger.warning(
-                f"[WORKFLOW]: Queue is empty for : {task.workflow.name!r} : "
-                f"{task.runner.cron}"
+        # VALIDATE: Check this task is the first release in the queue or not.
+        current_date: datetime = current_date.replace(second=0, microsecond=0)
+        if (first_date := q.first_queue.date) != current_date:
+            logger.debug(
+                f"[WORKFLOW]: Skip schedule "
+                f"{first_date:%Y-%m-%d %H:%M:%S} "
+                f"for : {task.alias!r} : {task.runner.cron}"
             )
             continue
 
-        # NOTE: Remove this datetime from queue.
-        queue[task.alias].pop(0)
+        # NOTE: Pop the latest release and push it to running.
+        release: WorkflowRelease = heappop(q.queue)
+        q.push_running(release)
+
+        logger.info(
+            f"[WORKFLOW]: Start thread: '{task.alias}|{str(task.runner.cron)}|"
+            f"{release.date:%Y%m%d%H%M}'"
+        )
 
         # NOTE: Create thread name that able to tracking with observe schedule
         #   job.
         thread_name: str = (
-            f"{task.workflow.name}|{str(task.runner.cron)}|"
-            f"{task.runner.date:%Y%m%d%H%M}"
+            f"{task.alias}|{str(task.runner.cron)}|"
+            f"{release.date:%Y%m%d%H%M}"
         )
 
         wf_thread: Thread = Thread(
             target=catch_exceptions(cancel_on_failure=True)(task.release),
-            kwargs={
-                "queue": queue,
-                "running": running,
-            },
+            kwargs={"release": release, "queue": q, "log": log},
             name=thread_name,
             daemon=True,
         )
@@ -419,20 +435,21 @@ def workflow_task_release(
 
         delay()
 
-    logger.debug(f"[WORKFLOW]: {'=' * 100}")
+    logger.debug(f"[SCHEDULE]: End schedule release {'=' * 80}")
 
 
-def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
-    """Workflow schedule for monitoring long running thread from the schedule
-    control.
+def monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
+    """Monitoring function that running every five minute for track long running
+    thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
-    :rtype: None
+    :type threads: dict[str, Thread]
     """
     logger.debug(
         "[MONITOR]: Start checking long running workflow release task."
     )
-    snapshot_threads = list(threads.keys())
+
+    snapshot_threads: list[str] = list(threads.keys())
     for t_name in snapshot_threads:
 
         # NOTE: remove the thread that running success.
@@ -440,18 +457,23 @@ def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
             threads.pop(t_name)
 
 
-def workflow_control(
+def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
     externals: DictData | None = None,
+    *,
+    log: type[Log] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Workflow scheduler control.
+    """Scheduler control function that running every minute.
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: An datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
+    :param log:
+
     :rtype: list[str]
     """
+    # NOTE: Lazy import Scheduler object from the schedule package.
     try:
         from schedule import Scheduler
     except ImportError:
@@ -459,30 +481,27 @@ def workflow_control(
             "Should install schedule package before use this module."
         ) from None
 
+    log: type[Log] = log or FileLog
     scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
+    stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
 
-    # NOTE: Design workflow queue caching.
-    #   ---
-    #   {"workflow-name": [<release-datetime>, <release-datetime>, ...]}
-    #
-    wf_queue: dict[str, list[datetime]] = {}
-    thread_releases: dict[str, Thread] = {}
+    # IMPORTANT: Create main mapping of queue and thread object.
+    queue: dict[str, WorkflowQueue] = {}
+    threads: dict[str, Thread] = {}
 
-    start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
+    start_date_waiting: datetime = start_date.replace(
         second=0, microsecond=0
-    )
+    ) + timedelta(minutes=1)
 
-    # NOTE: Create pair of workflow and on from schedule model.
-    workflow_tasks: list[WorkflowTaskData] = []
+    # NOTE: Start create workflow tasks from list of schedule name.
+    tasks: list[WorkflowTask] = []
     for name in schedules:
         schedule: Schedule = Schedule.from_loader(name, externals=externals)
-
-        # NOTE: Create a workflow task data instance from schedule object.
-        workflow_tasks.extend(
+        tasks.extend(
             schedule.tasks(
                 start_date_waiting,
-                queue=wf_queue,
+                queue=queue,
                 externals=externals,
             ),
         )
@@ -492,23 +511,33 @@ def workflow_control(
         scheduler.every(1)
         .minutes.at(":02")
         .do(
-            workflow_task_release,
-            workflow_tasks=workflow_tasks,
-            stop=(stop or (start_date + config.stop_boundary_delta)),
-            queue=wf_queue,
-            threads=thread_releases,
+            schedule_task,
+            tasks=tasks,
+            stop=stop_date,
+            queue=queue,
+            threads=threads,
+            log=log,
         )
         .tag("control")
     )
 
     # NOTE: Checking zombie task with schedule job will start every 5 minute.
-    scheduler.every(5).minutes.at(":10").do(
-        workflow_monitor,
-        threads=thread_releases,
-    ).tag("monitor")
+    (
+        scheduler.every(5)
+        .minutes.at(":10")
+        .do(
+            monitor,
+            threads=threads,
+        )
+        .tag("monitor")
+    )
 
     # NOTE: Start running schedule
-    logger.info(f"[WORKFLOW]: Start schedule: {schedules}")
+    logger.info(
+        f"[SCHEDULE]: Schedule: {schedules} with stopper: "
+        f"{stop_date:%Y-%m-%d %H:%M:%S}"
+    )
+
     while True:
         scheduler.run_pending()
         time.sleep(1)
@@ -516,29 +545,35 @@ def workflow_control(
         # NOTE: Break the scheduler when the control job does not exists.
         if not scheduler.get_jobs("control"):
             scheduler.clear("monitor")
-            logger.warning(
-                f"[WORKFLOW]: Workflow release thread: {thread_releases}"
-            )
-            logger.warning("[WORKFLOW]: Does not have any schedule jobs !!!")
+
+            while len(threads) > 0:
+                logger.warning(
+                    "[SCHEDULE]: Waiting schedule release thread that still "
+                    "running in background."
+                )
+                delay(15)
+                monitor(threads)
+
             break
 
     logger.warning(
-        f"Queue: {[list(queue2str(wf_queue[wf])) for wf in wf_queue]}"
+        f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
     )
     return schedules
 
 
-def workflow_runner(
+def schedule_runner(
     stop: datetime | None = None,
     externals: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> list[str]:  # pragma: no cov
-    """Workflow application that running multiprocessing schedule with chunk of
-    workflows that exists in config path.
+    """Schedule runner function for start submit the ``schedule_control`` func
+    in multiprocessing pool with chunk of schedule config that exists in config
+    path by ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS``.
 
     :param stop: A stop datetime object that force stop running scheduler.
-    :param excluded:
     :param externals:
+    :param excluded: A list of schedule name that want to excluded from finding.
 
     :rtype: list[str]
 
@@ -549,24 +584,21 @@ def workflow_runner(
 
         The current workflow logic that split to process will be below diagram:
 
-        PIPELINES ==> process 01 ==> schedule --> thread of release
-                                                  workflow task 01 01
-                                              --> thread of release
-                                                  workflow task 01 02
-                  ==> process 02 ==> schedule --> thread of release
-                                                  workflow task 02 01
-                                              --> thread of release
-                                                  workflow task 02 02
-                  ==> ...
+        MAIN ==> process 01 ==> schedule --> thread of release task 01 01
+                                        --> thread of release task 01 02
+                            ==> schedule --> thread of release task 02 01
+                                        --> thread of release task 02 02
+            ==> process 02
     """
-    excluded: list[str] = excluded or []
+    results: list[str] = []
 
     with ProcessPoolExecutor(
         max_workers=config.max_schedule_process,
     ) as executor:
+
         futures: list[Future] = [
             executor.submit(
-                workflow_control,
+                schedule_control,
                 schedules=[load[0] for load in loader],
                 stop=stop,
                 externals=(externals or {}),
@@ -577,10 +609,13 @@ def workflow_runner(
             )
         ]
 
-        results: list[str] = []
         for future in as_completed(futures):
+
+            # NOTE: Raise error when it has any error from schedule_control.
             if err := future.exception():
                 logger.error(str(err))
                 raise WorkflowException(str(err)) from err
+
             results.extend(future.result(timeout=1))
-        return results
+
+    return results

ddeutil/workflow/stage.py

@@ -346,6 +346,11 @@ class EmptyStage(BaseStage):
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
+            if self.sleep > 30:
+                logger.info(
+                    f"({cut_id(run_id)}) [STAGE]: ... sleep "
+                    f"({self.sleep} seconds)"
+                )
             time.sleep(self.sleep)
         return Result(status=0, context={}, run_id=run_id)
 

ddeutil/workflow/utils.py

@@ -577,10 +577,6 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         yield chain((first_el,), chunk_it)
 
 
-def queue2str(queue: list[datetime]) -> Iterator[str]:  # pragma: no cov
-    return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
-
-
 def cut_id(run_id: str, *, num: int = 6):
     """Cutting running ID with length.
 

ddeutil/workflow/workflow.py

@@ -43,16 +43,15 @@ from typing_extensions import Self
 from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
 from .conf import FileLog, Loader, Log, config, get_logger
+from .cron import On
 from .exceptions import JobException, WorkflowException
 from .job import Job
-from .on import On
 from .params import Param
 from .result import Result
 from .utils import (
     cut_id,
     delay,
     gen_id,
-    get_diff_sec,
     get_dt_now,
     has_template,
     param2template,
@@ -64,7 +63,7 @@ __all__: TupleStr = (
     "Workflow",
     "WorkflowRelease",
     "WorkflowQueue",
-    "WorkflowTaskData",
+    "WorkflowTask",
 )
 
 
@@ -168,7 +167,23 @@ class WorkflowQueue:
         """
         return len(self.queue) > 0
 
-    def check_queue(self, value: WorkflowRelease) -> bool:
+    @property
+    def first_queue(self) -> WorkflowRelease:
+        """Check an input WorkflowRelease object is the first value of the
+        waiting queue.
+
+        :rtype: bool
+        """
+        # NOTE: Old logic to peeking the first release from waiting queue.
+        #
+        # first_value: WorkflowRelease = heappop(self.queue)
+        # heappush(self.queue, first_value)
+        #
+        # return first_value
+        #
+        return self.queue[0]
+
+    def check_queue(self, value: WorkflowRelease | datetime) -> bool:
         """Check a WorkflowRelease value already exists in list of tracking
         queues.
 
@@ -177,6 +192,9 @@ class WorkflowQueue:
 
         :rtype: bool
         """
+        if isinstance(value, datetime):
+            value = WorkflowRelease.from_dt(value)
+
         return (
             (value in self.queue)
             or (value in self.running)
@@ -184,20 +202,37 @@ class WorkflowQueue:
         )
 
     def push_queue(self, value: WorkflowRelease) -> Self:
-        """Push data to the queue."""
+        """Push data to the waiting queue."""
         heappush(self.queue, value)
         return self
 
     def push_running(self, value: WorkflowRelease) -> Self:
-        """Push data to the running."""
+        """Push WorkflowRelease to the running queue."""
         heappush(self.running, value)
         return self
 
     def remove_running(self, value: WorkflowRelease) -> Self:
-        """Remove data on the running if it exists."""
+        """Remove WorkflowRelease in the running queue if it exists."""
         if value in self.running:
             self.running.remove(value)
 
+    def push_complete(self, value: WorkflowRelease) -> Self:
+        """Push WorkflowRelease to the complete queue."""
+        heappush(self.complete, value)
+
+        # NOTE: Remove complete queue on workflow that keep more than the
+        #   maximum config.
+        num_complete_delete: int = (
+            len(self.complete) - config.max_queue_complete_hist
+        )
+
+        if num_complete_delete > 0:
+            print(num_complete_delete)
+            for _ in range(num_complete_delete):
+                heappop(self.complete)
+
+        return self
+
 
 class Workflow(BaseModel):
     """Workflow Pydantic model.
@@ -448,6 +483,7 @@ class Workflow(BaseModel):
         queue: (
             WorkflowQueue | list[datetime] | list[WorkflowRelease] | None
         ) = None,
+        override_log_name: str | None = None,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
         release templating that include logical date (release date), execution
@@ -462,11 +498,14 @@ class Workflow(BaseModel):
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
         :param queue: A WorkflowQueue object.
+        :param override_log_name: An override logging name that use instead
+            the workflow name.
 
         :rtype: Result
         """
         log: type[Log] = log or FileLog
-        run_id: str = run_id or gen_id(self.name, unique=True)
+        name: str = override_log_name or self.name
+        run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
 
         # VALIDATE: Change queue value to WorkflowQueue object.
@@ -478,7 +517,7 @@ class Workflow(BaseModel):
             release: WorkflowRelease = WorkflowRelease.from_dt(release)
 
         logger.debug(
-            f"({cut_id(run_id)}) [RELEASE]: {self.name!r} : Start release - "
+            f"({cut_id(run_id)}) [RELEASE]: Start release - {name!r} : "
             f"{release.date:%Y-%m-%d %H:%M:%S}"
         )
 
@@ -499,14 +538,14 @@ class Workflow(BaseModel):
             run_id=run_id,
         )
         logger.debug(
-            f"({cut_id(run_id)}) [RELEASE]: {self.name!r} : End release - "
+            f"({cut_id(run_id)}) [RELEASE]: End release - {name!r} : "
             f"{release.date:%Y-%m-%d %H:%M:%S}"
         )
 
         rs.set_parent_run_id(run_id)
         rs_log: Log = log.model_validate(
             {
-                "name": self.name,
+                "name": name,
                 "release": release.date,
                 "type": release.type,
                 "context": rs.context,
@@ -520,7 +559,7 @@ class Workflow(BaseModel):
 
         # NOTE: Remove this release from running.
         queue.remove_running(release)
-        heappush(queue.complete, release)
+        queue.push_complete(release)
 
         context: dict[str, Any] = rs.context
         context.pop("params")
@@ -534,7 +573,7 @@ class Workflow(BaseModel):
             },
         )
 
-    def queue_poking(
+    def queue(
         self,
         offset: float,
         end_date: datetime,
@@ -667,7 +706,7 @@ class Workflow(BaseModel):
         wf_queue: WorkflowQueue = WorkflowQueue()
 
         # NOTE: Make queue to the workflow queue object.
-        self.queue_poking(
+        self.queue(
             offset,
             end_date=end_date,
             queue=wf_queue,
@@ -707,7 +746,7 @@ class Workflow(BaseModel):
 
                     # WARNING: I already call queue poking again because issue
                     #   about the every minute crontab.
-                    self.queue_poking(
+                    self.queue(
                         offset,
                         end_date,
                         queue=wf_queue,
@@ -729,7 +768,7 @@ class Workflow(BaseModel):
                     )
                 )
 
-                self.queue_poking(
+                self.queue(
                     offset,
                     end_date,
                     queue=wf_queue,
@@ -1074,7 +1113,7 @@ class Workflow(BaseModel):
 
 
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
-class WorkflowTaskData:
+class WorkflowTask:
     """Workflow task Pydantic dataclass object that use to keep mapping data and
     workflow model for passing to the multithreading task.
 
@@ -1085,151 +1124,95 @@ class WorkflowTaskData:
     alias: str
     workflow: Workflow
     runner: CronRunner
-    params: DictData = field(default_factory=dict)
+    values: DictData = field(default_factory=dict)
 
     def release(
         self,
-        queue: dict[str, list[datetime]],
-        log: Log | None = None,
+        release: datetime | WorkflowRelease | None = None,
         run_id: str | None = None,
-        *,
-        waiting_sec: int = 60,
-        sleep_interval: int = 15,
-    ) -> Result:  # pragma: no cov
-        """Release the workflow task data that use the same logic of
-        `workflow.release` method but use different the queue object for
-        tracking release datetime to run.
-
-        :param queue: A mapping of alias name and list of release datetime.
-        :param log: A log object for saving result logging from workflow
-            execution process.
+        log: type[Log] = None,
+        queue: (
+            WorkflowQueue | list[datetime] | list[WorkflowRelease] | None
+        ) = None,
+    ) -> Result:
+        """Release the workflow task data.
+
+        :param release: A release datetime or WorkflowRelease object.
         :param run_id: A workflow running ID for this release.
-        :param waiting_sec: A second period value that allow workflow execute.
-        :param sleep_interval: A second value that want to waiting until time
-            to execute.
+        :param log: A log class that want to save the execution result.
+        :param queue: A WorkflowQueue object.
 
         :rtype: Result
         """
         log: type[Log] = log or FileLog
-        run_id: str = run_id or gen_id(self.alias, unique=True)
-        rs_release: Result = Result(run_id=run_id)
-        runner: CronRunner = self.runner
-
-        # NOTE: get next schedule time that generate from now.
-        next_time: datetime = runner.date
-
-        # NOTE: get next utils it does not running.
-        while log.is_pointed(self.workflow.name, next_time) or (
-            next_time in queue[self.alias]
-        ):
-            next_time: datetime = runner.next
-
-        logger.debug(
-            f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
-            f"{runner.cron} : {next_time:%Y-%m-%d %H:%M:%S}"
+        return self.workflow.release(
+            release=release or self.runner.next,
+            params=self.values,
+            run_id=run_id,
+            log=log,
+            queue=queue,
+            override_log_name=self.alias,
         )
-        heappush(queue[self.alias], next_time)
-        start_sec: float = time.monotonic()
-
-        if get_diff_sec(next_time, tz=runner.tz) > waiting_sec:
-            logger.debug(
-                f"({cut_id(run_id)}) [WORKFLOW]: {self.workflow.name!r} : "
-                f"{runner.cron} "
-                f": Does not closely >> {next_time:%Y-%m-%d %H:%M:%S}"
-            )
 
-            # NOTE: Add this next running datetime that not in period to queue
-            #   and remove it to running.
-            queue[self.alias].remove(next_time)
+    def queue(
+        self,
+        end_date: datetime,
+        queue: WorkflowQueue,
+        log: type[Log],
+        *,
+        force_run: bool = False,
+    ):
+        """Generate WorkflowRelease to WorkflowQueue object.
+
+        :param end_date: An end datetime object.
+        :param queue: A workflow queue object.
+        :param log: A log class that want to making log object.
+        :param force_run: A flag that allow to release workflow if the log with
+            that release was pointed.
 
-            time.sleep(0.2)
-            return rs_release.catch(status=0, context={})
+        :rtype: WorkflowQueue
+        """
+        if self.runner.date > end_date:
+            return queue
 
-        logger.debug(
-            f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
-            f"{runner.cron} : Closely to run >> {next_time:%Y-%m-%d %H:%M:%S}"
+        workflow_release = WorkflowRelease(
+            date=self.runner.date,
+            offset=0,
+            end_date=end_date,
+            runner=self.runner,
+            type="task",
         )
 
-        # NOTE: Release when the time is nearly to schedule time.
-        while (duration := get_diff_sec(next_time, tz=config.tz)) > (
-            sleep_interval + 5
+        while queue.check_queue(workflow_release) or (
+            log.is_pointed(name=self.alias, release=workflow_release.date)
+            and not force_run
         ):
-            logger.debug(
-                f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
-                f"{runner.cron} : Sleep until: {duration}"
+            workflow_release = WorkflowRelease(
+                date=self.runner.next,
+                offset=0,
+                end_date=end_date,
+                runner=self.runner,
+                type="task",
             )
-            time.sleep(15)
 
-        time.sleep(0.5)
+        if self.runner.date > end_date:
+            return queue
 
-        # NOTE: Release parameter that use to change if params has
-        #   templating.
-        release_params: DictData = {
-            "release": {
-                "logical_date": next_time,
-                "execute_date": datetime.now(tz=config.tz),
-                "run_id": run_id,
-                "timezone": runner.tz,
-            },
-        }
+        # NOTE: Push the WorkflowRelease object to queue.
+        queue.push_queue(workflow_release)
 
-        # WARNING: Re-create workflow object that use new running workflow ID.
-        rs: Result = self.workflow.execute(
-            params=param2template(self.params, release_params),
-        )
-        logger.debug(
-            f"({cut_id(run_id)}) [CORE]: {self.workflow.name!r} : "
-            f"{runner.cron} : End release - {next_time:%Y-%m-%d %H:%M:%S}"
-        )
-
-        # NOTE: Set parent ID on this result.
-        rs.set_parent_run_id(run_id)
-
-        # NOTE: Save result to log object saving.
-        rs_log: Log = log.model_validate(
-            {
-                "name": self.workflow.name,
-                "type": "schedule",
-                "release": next_time,
-                "context": rs.context,
-                "parent_run_id": rs.run_id,
-                "run_id": rs.run_id,
-            }
-        )
-        rs_log.save(excluded=None)
-
-        # NOTE: Remove the current release date from the running.
-        queue[self.alias].remove(next_time)
-        total_sec: float = time.monotonic() - start_sec
-
-        # IMPORTANT:
-        #   Add the next running datetime to workflow task queue.
-        future_running_time: datetime = runner.next
-
-        while (
-            future_running_time in queue[self.alias]
-            or (future_running_time - next_time).total_seconds() < total_sec
-        ):  # pragma: no cov
-            future_running_time: datetime = runner.next
-
-        # NOTE: Queue next release date.
-        logger.debug(f"[CORE]: {'-' * 100}")
-
-        context: dict[str, Any] = rs.context
-        context.pop("params")
+        return queue
 
-        return rs_release.catch(
-            status=0,
-            context={
-                "params": self.params,
-                "release": {"status": "success", "logical_date": next_time},
-                "outputs": context,
-            },
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(alias={self.alias!r}, "
+            f"workflow={self.workflow.name!r}, runner={self.runner!r}, "
+            f"values={self.values})"
         )
 
-    def __eq__(self, other: WorkflowTaskData) -> bool:
+    def __eq__(self, other: WorkflowTask) -> bool:
         """Override equal property that will compare only the same type."""
-        if isinstance(other, WorkflowTaskData):
+        if isinstance(other, WorkflowTask):
             return (
                 self.workflow.name == other.workflow.name
                 and self.runner.cron == other.runner.cron

{ddeutil_workflow-0.0.23.dist-info → ddeutil_workflow-0.0.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.23
+Version: 0.0.24
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,7 +24,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ddeutil>=0.4.3
 Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
-Requires-Dist: pydantic==2.10.2
+Requires-Dist: pydantic==2.10.4
 Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: typer==0.15.1
 Requires-Dist: schedule<2.0.0,==1.2.2
@@ -196,7 +196,8 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
 | `WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT`    | Core      | 600                              |                                                                                                                    |        |
-| `WORKFLOW_CORE_MAX_ON_PER_WORKFLOW`     | Core      | 5                                |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW`   | Core      | 5                                |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST` | Core      | 16                               |                                                                                                                    |        |
 | `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |

ddeutil_workflow-0.0.24.dist-info/RECORD

@@ -0,0 +1,24 @@
+ddeutil/workflow/__about__.py,sha256=LbAkk7O3dezpuJ-KPhsDQuHdrO9T0qmhBd-oDJzBhq4,28
+ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
+ddeutil/workflow/__init__.py,sha256=49eGrCuchPVZKMybRouAviNhbulK_F6VwCmLm76hIss,1478
+ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
+ddeutil/workflow/api.py,sha256=cdRxqwVyGm_Ni_OmflIP35vUkkq8lHpF3xHh_BvVrKs,4692
+ddeutil/workflow/cli.py,sha256=8C5Xri1_82B-sxQcKMPRjDJcuYJG3FZ2bJehvs_xZ4s,3278
+ddeutil/workflow/conf.py,sha256=Al-00Uru2fCJaW2C_vt4IFuBDpI8Y5C4oAuLJ0Vdvbk,16110
+ddeutil/workflow/cron.py,sha256=0SxC3SH-8V1idgAEFOY-gYFEQPjK_zymmc5XqPoX_0I,7504
+ddeutil/workflow/exceptions.py,sha256=NqnQJP52S59XIYMeXbTDbr4xH2UZ5EA3ejpU5Z4g6cQ,894
+ddeutil/workflow/job.py,sha256=cvSLMdc1sMl1MeU7so7Oe2SdRYxQwt6hm55mLV1iP-Y,24219
+ddeutil/workflow/params.py,sha256=uPGkZx18E-iZ8BteqQ2ONgg0frhF3ZmP5cOyfK2j59U,5280
+ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
+ddeutil/workflow/result.py,sha256=WIC8MsnfLiWNpZomT6jS4YCdYhlbIVVBjtGGe2dkoKk,3404
+ddeutil/workflow/route.py,sha256=bH5IT90JVjCDe9A0gIefpQQBEfcd-o1uCHE9AvNglvU,6754
+ddeutil/workflow/scheduler.py,sha256=UI8wK2xBYmM3Bh_hel0TMzuJWyezM83Yn4xoiYqTSSQ,20238
+ddeutil/workflow/stage.py,sha256=a2sngzs9DkP6GU2pgAD3QvGoijyBQTR_pOhyJUIuWAo,26692
+ddeutil/workflow/utils.py,sha256=PhNJ54oKnZfq4nVOeP3tDjFN43ArUsMOnpcbSu7bo4I,18450
+ddeutil/workflow/workflow.py,sha256=JyT65Tql7CueQn2z4ZGhp6r44jgYDMcCOpxhiwI19uM,41403
+ddeutil_workflow-0.0.24.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.24.dist-info/METADATA,sha256=0yh6zKsIu1COnhl-25rOxBGEqLQbrJZzA0IhriO3XwA,14234
+ddeutil_workflow-0.0.24.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ddeutil_workflow-0.0.24.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.24.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.24.dist-info/RECORD,,

ddeutil_workflow-0.0.23.dist-info/RECORD

@@ -1,24 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=xc6s739CYU9opnZq2_D6Mx3ekymsbYOQKM2zsgw85oc,28
-ddeutil/workflow/__cron.py,sha256=_2P9nmGOwGdv5bLgf9TpML2HBgqLv_qRgiO1Rulo1PA,26693
-ddeutil/workflow/__init__.py,sha256=dH3T06kO8aEiRHAiL-d8a3IvOS0Fx80lS3AWz6rGdQk,1443
-ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
-ddeutil/workflow/api.py,sha256=ceTJfjXIw-3cgw4yx2QCcGLuA3STb0o7ELcVR_tfZFI,4700
-ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
-ddeutil/workflow/conf.py,sha256=GsbuJDQfQoAGiR4keUEoB4lKfZxdkaiZ4N4FfIHc0xY,15814
-ddeutil/workflow/exceptions.py,sha256=NqnQJP52S59XIYMeXbTDbr4xH2UZ5EA3ejpU5Z4g6cQ,894
-ddeutil/workflow/job.py,sha256=cvSLMdc1sMl1MeU7so7Oe2SdRYxQwt6hm55mLV1iP-Y,24219
-ddeutil/workflow/on.py,sha256=0SxC3SH-8V1idgAEFOY-gYFEQPjK_zymmc5XqPoX_0I,7504
-ddeutil/workflow/params.py,sha256=uPGkZx18E-iZ8BteqQ2ONgg0frhF3ZmP5cOyfK2j59U,5280
-ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
-ddeutil/workflow/result.py,sha256=WIC8MsnfLiWNpZomT6jS4YCdYhlbIVVBjtGGe2dkoKk,3404
-ddeutil/workflow/route.py,sha256=bH5IT90JVjCDe9A0gIefpQQBEfcd-o1uCHE9AvNglvU,6754
-ddeutil/workflow/scheduler.py,sha256=MYHf1bz8nsT8tJYcXgC-UycWbJ56Hx_zXwUAwWICimM,19141
-ddeutil/workflow/stage.py,sha256=y6gjNzQy7xAM0n-lwqAEoC4x0lopH0K-Y77a_gvq4t8,26505
-ddeutil/workflow/utils.py,sha256=ZVQh5vArWHNfCFYWYjHvkVD5aH-350ycfcZxDewELHM,18578
-ddeutil/workflow/workflow.py,sha256=jmJxAEkczuanmj041OLgeBUDz9Y8XfFNVP_B3Xi7QDY,42557
-ddeutil_workflow-0.0.23.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.23.dist-info/METADATA,sha256=4KqKSGCbc8dWHwWSCSgU4m_wsOouFHsiCGtdyo0Mf5U,14017
-ddeutil_workflow-0.0.23.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ddeutil_workflow-0.0.23.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
-ddeutil_workflow-0.0.23.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.23.dist-info/RECORD,,