ddeutil-workflow 0.0.13__py3-none-any.whl → 0.0.14__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -11,34 +11,39 @@ import json
 import logging
 import os
 import time
-from collections.abc import Iterator
 from concurrent.futures import (
     Future,
     ProcessPoolExecutor,
     ThreadPoolExecutor,
     as_completed,
 )
-from dataclasses import dataclass, field
+from dataclasses import field
 from datetime import datetime, timedelta
 from functools import wraps
 from heapq import heappush
 from queue import Queue
 from textwrap import dedent
 from threading import Thread
-from typing import Optional
+from typing import Callable, Optional
 from zoneinfo import ZoneInfo
 
-from dotenv import load_dotenv
 from pydantic import BaseModel, Field
+from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
+try:
+    from typing import ParamSpec
+except ImportError:
+    from typing_extensions import ParamSpec
+
 try:
     from schedule import CancelJob
 except ImportError:
     CancelJob = None
 
 from .__types import DictData, TupleStr
+from .conf import config
 from .cron import CronRunner
 from .exceptions import JobException, WorkflowException
 from .job import Job
@@ -54,9 +59,10 @@ from .utils import (
     get_diff_sec,
     has_template,
     param2template,
+    queue2str,
 )
 
-load_dotenv()
+P = ParamSpec("P")
 logger = get_logger("ddeutil.workflow")
 
 # NOTE: Adjust logging level on the schedule package.
@@ -65,9 +71,9 @@ logging.getLogger("schedule").setLevel(logging.INFO)
 
 __all__: TupleStr = (
     "Workflow",
-    "WorkflowSchedule",
-    "WorkflowTask",
+    "WorkflowTaskData",
     "Schedule",
+    "ScheduleWorkflow",
     "workflow_task",
     "workflow_long_running_task",
     "workflow_control",
@@ -76,10 +82,10 @@ __all__: TupleStr = (
 
 
 class Workflow(BaseModel):
-    """Workflow Model this is the main future of this project because it use to
-    be workflow data for running everywhere that you want or using it to
-    scheduler task in background. It use lightweight coding line from Pydantic
-    Model and enhance execute method on it.
+    """Workflow Pydantic Model this is the main future of this project because
+    it use to be workflow data for running everywhere that you want or using it
+    to scheduler task in background. It use lightweight coding line from
+    Pydantic Model and enhance execute method on it.
     """
 
     name: str = Field(description="A workflow name.")
@@ -91,7 +97,7 @@ class Workflow(BaseModel):
     )
     params: dict[str, Param] = Field(
         default_factory=dict,
-        description="A parameters that want to use on this workflow.",
+        description="A parameters that need to use on this workflow.",
     )
     on: list[On] = Field(
         default_factory=list,
@@ -103,14 +109,19 @@ class Workflow(BaseModel):
     )
     run_id: Optional[str] = Field(
         default=None,
-        description="A running workflow ID.",
+        description=(
+            "A running workflow ID that is able to change after initialize."
+        ),
         repr=False,
         exclude=True,
     )
 
     @property
     def new_run_id(self) -> str:
-        """Running ID of this workflow that always generate new unique value."""
+        """Running ID of this workflow that always generate new unique value.
+
+        :rtype: str
+        """
         return gen_id(self.name, unique=True)
 
     @classmethod
@@ -144,8 +155,17 @@ class Workflow(BaseModel):
         return cls.model_validate(obj=loader_data)
 
     @classmethod
-    def __bypass_on(cls, data: DictData, externals: DictData | None = None):
-        """Bypass the on data to loaded config data."""
+    def __bypass_on(
+        cls,
+        data: DictData,
+        externals: DictData | None = None,
+    ) -> DictData:
+        """Bypass the on data to loaded config data.
+
+        :param data:
+        :param externals:
+        :rtype: DictData
+        """
         if on := data.pop("on", []):
             if isinstance(on, str):
                 on = [on]
@@ -180,12 +200,18 @@ class Workflow(BaseModel):
 
     @field_validator("desc", mode="after")
     def ___prepare_desc(cls, value: str) -> str:
-        """Prepare description string that was created on a template."""
+        """Prepare description string that was created on a template.
+
+        :rtype: str
+        """
         return dedent(value)
 
     @model_validator(mode="after")
-    def __validate_jobs_need_and_prepare_running_id(self):
-        """Validate each need job in any jobs should exists."""
+    def __validate_jobs_need_and_prepare_running_id(self) -> Self:
+        """Validate each need job in any jobs should exists.
+
+        :rtype: Self
+        """
         for job in self.jobs:
             if not_exist := [
                 need for need in self.jobs[job].needs if need not in self.jobs
@@ -221,7 +247,7 @@ class Workflow(BaseModel):
         return self.model_copy(update={"run_id": run_id})
 
     def job(self, name: str) -> Job:
-        """Return Job model that exists on this workflow.
+        """Return this workflow's job that already created on this job field.
 
         :param name: A job name that want to get from a mapping of job models.
         :type name: str
@@ -237,11 +263,18 @@ class Workflow(BaseModel):
         return self.jobs[name]
 
     def parameterize(self, params: DictData) -> DictData:
-        """Prepare parameters before passing to execution process. This method
-        will create jobs key to params mapping that will keep any result from
-        job execution.
+        """Prepare a passing parameters before use it in execution process.
+        This method will validate keys of an incoming params with this object
+        necessary params field and then create a jobs key to result mapping
+        that will keep any execution result from its job.
+
+            ... {
+            ...     "params": <an-incoming-params>,
+            ...     "jobs": {}
+            ... }
 
         :param params: A parameter mapping that receive from workflow execution.
+        :type params: DictData
         :rtype: DictData
         """
         # VALIDATE: Incoming params should have keys that set on this workflow.
@@ -255,7 +288,7 @@ class Workflow(BaseModel):
                 f"{', '.join(check_key)}."
             )
 
-        # NOTE: mapping type of param before adding it to params variable.
+        # NOTE: Mapping type of param before adding it to the ``params`` key.
         return {
             "params": (
                 params
@@ -299,9 +332,8 @@ class Workflow(BaseModel):
             f"queue id: {id(queue)}"
         )
         log: Log = log or FileLog
-        tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
         gen: CronRunner = on.generate(
-            datetime.now(tz=tz).replace(second=0, microsecond=0)
+            datetime.now(tz=config.tz).replace(second=0, microsecond=0)
             + timedelta(seconds=1)
         )
         cron_tz: ZoneInfo = gen.tz
@@ -456,35 +488,55 @@ class Workflow(BaseModel):
 
     def execute_job(
         self,
-        job: str,
+        job_id: str,
         params: DictData,
+        *,
+        raise_error: bool = True,
     ) -> Result:
-        """Job Executor that use on workflow executor.
+        """Workflow Job execution with passing dynamic parameters from the
+        workflow execution to the target job.
 
-        :param job: A job ID that want to execute.
+            This execution is the minimum level of execution of this workflow
+        model. It different with ``self.execute`` because this method run only
+        one job and return with context of this job data.
+
+        :param job_id: A job ID that want to execute.
         :param params: A params that was parameterized from workflow execution.
+        :param raise_error: A flag that raise error instead catching to result
+            if it get exception from job execution.
         :rtype: Result
         """
         # VALIDATE: check a job ID that exists in this workflow or not.
-        if job not in self.jobs:
+        if job_id not in self.jobs:
             raise WorkflowException(
-                f"The job ID: {job} does not exists on {self.name!r} workflow."
+                f"The job ID: {job_id} does not exists in {self.name!r} "
+                f"workflow."
             )
-        try:
-            logger.info(f"({self.run_id}) [WORKFLOW]: Start execute: {job!r}")
 
-            # IMPORTANT:
-            #   Change any job running IDs to this workflow running ID.
-            job_obj: Job = self.jobs[job].get_running_id(self.run_id)
-            j_rs: Result = job_obj.execute(params=params)
+        context: DictData = {}
+        logger.info(f"({self.run_id}) [WORKFLOW]: Start execute: {job_id!r}")
 
+        # IMPORTANT:
+        #   Change any job running IDs to this workflow running ID.
+        #
+        try:
+            job: Job = self.jobs[job_id].get_running_id(self.run_id)
+            job.set_outputs(
+                job.execute(params=params).context,
+                to=context,
+            )
         except JobException as err:
-            raise WorkflowException(f"{job}: JobException: {err}") from None
+            logger.error(
+                f"({self.run_id}) [WORKFLOW]: {err.__class__.__name__}: {err}"
+            )
+            if raise_error:
+                raise WorkflowException(
+                    f"Get job execution error {job_id}: JobException: {err}"
+                ) from None
+            else:
+                raise NotImplementedError() from None
 
-        return Result(
-            status=j_rs.status,
-            context={job: job_obj.set_outputs(j_rs.context)},
-        )
+        return Result(status=0, context=context)
 
     def execute(
         self,
@@ -492,17 +544,8 @@ class Workflow(BaseModel):
         *,
         timeout: int = 60,
     ) -> Result:
-        """Execute workflow with passing dynamic parameters to any jobs that
-        included in the workflow.
-
-        :param params: An input parameters that use on workflow execution that
-            will parameterize before using it.
-        :param timeout: A workflow execution time out in second unit that use
-            for limit time of execution and waiting job dependency.
-        :rtype: Result
-
-        See Also:
-        ---
+        """Execute workflow with passing a dynamic parameters to all jobs that
+        included in this workflow model with ``jobs`` field.
 
             The result of execution process for each jobs and stages on this
         workflow will keeping in dict which able to catch out with all jobs and
@@ -513,10 +556,22 @@ class Workflow(BaseModel):
 
             ... ${job-name}.stages.${stage-id}.outputs.${key}
 
+        :param params: An input parameters that use on workflow execution that
+            will parameterize before using it. Default is None.
+        :type params: DictData | None
+        :param timeout: A workflow execution time out in second unit that use
+            for limit time of execution and waiting job dependency. Default is
+            60 seconds.
+        :type timeout: int
+        :rtype: Result
         """
         logger.info(f"({self.run_id}) [CORE]: Start Execute: {self.name!r} ...")
-        params: DictData = params or {}
+
+        # NOTE: I use this condition because this method allow passing empty
+        #   params and I do not want to create new dict object.
+        params: DictData = {} if params is None else params
         ts: float = time.monotonic()
+        rs: Result = Result()
 
         # NOTE: It should not do anything if it does not have job.
         if not self.jobs:
@@ -524,7 +579,7 @@ class Workflow(BaseModel):
                 f"({self.run_id}) [WORKFLOW]: This workflow: {self.name!r} "
                 f"does not have any jobs"
             )
-            return Result(status=0, context=params)
+            return rs.catch(status=0, context=params)
 
         # NOTE: Create a job queue that keep the job that want to running after
         #   it dependency condition.
@@ -535,21 +590,32 @@ class Workflow(BaseModel):
         # NOTE: Create result context that will pass this context to any
         #   execution dependency.
         context: DictData = self.parameterize(params)
+        status: int = 0
         try:
-            worker: int = int(os.getenv("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
-            (
-                self.__exec_non_threading(context, ts, jq, timeout=timeout)
-                if worker == 1
-                else self.__exec_threading(
-                    context, ts, jq, worker=worker, timeout=timeout
+            if config.max_job_parallel == 1:
+                self.__exec_non_threading(
+                    context=context,
+                    ts=ts,
+                    job_queue=jq,
+                    timeout=timeout,
+                )
+            else:
+                self.__exec_threading(
+                    context=context,
+                    ts=ts,
+                    job_queue=jq,
+                    worker=config.max_job_parallel,
+                    timeout=timeout,
                 )
-            )
-            return Result(status=0, context=context)
         except WorkflowException as err:
             context.update(
-                {"error_message": f"{err.__class__.__name__}: {err}"}
+                {
+                    "error": err,
+                    "error_message": f"{err.__class__.__name__}: {err}",
+                },
             )
-            return Result(status=1, context=context)
+            status = 1
+        return rs.catch(status=status, context=context)
 
     def __exec_threading(
         self,
@@ -560,11 +626,15 @@ class Workflow(BaseModel):
         worker: int = 2,
         timeout: int = 600,
     ) -> DictData:
-        """Workflow threading execution.
+        """Workflow execution by threading strategy.
+
+            If a job need dependency, it will check dependency job ID from
+        context data before allow it run.
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
             timeout.
+        :param job_queue: A job queue object.
         :param timeout: A second value unit that bounding running time.
         :param worker: A number of threading executor pool size.
         :rtype: DictData
@@ -598,18 +668,24 @@ class Workflow(BaseModel):
                         params=copy.deepcopy(context),
                     ),
                 )
+
+                # NOTE: Mark this job queue done.
                 job_queue.task_done()
 
             # NOTE: Wait for all items to finish processing
             job_queue.join()
 
-            for future in as_completed(futures):
+            for future in as_completed(futures, timeout=1800):
                 if err := future.exception():
                     logger.error(f"{err}")
                     raise WorkflowException(f"{err}")
-
-                # NOTE: Update job result to workflow result.
-                context["jobs"].update(future.result(timeout=20).conext)
+                try:
+                    # NOTE: Update job result to workflow result.
+                    context["jobs"].update(future.result(timeout=60).context)
+                except TimeoutError as err:
+                    raise WorkflowException(
+                        "Get result from future was timeout"
+                    ) from err
 
         if not_time_out_flag:
             return context
@@ -631,8 +707,11 @@ class Workflow(BaseModel):
         *,
         timeout: int = 600,
     ) -> DictData:
-        """Workflow non-threading execution that use sequential job running
-        and waiting previous run successful.
+        """Workflow execution with non-threading strategy that use sequential
+        job running and waiting previous job was run successful.
+
+            If a job need dependency, it will check dependency job ID from
+        context data before allow it run.
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
@@ -658,9 +737,14 @@ class Workflow(BaseModel):
                 time.sleep(0.25)
                 continue
 
-            # NOTE: Start job execution.
-            job_rs = self.execute_job(job_id, params=copy.deepcopy(context))
+            # NOTE: Start workflow job execution.
+            job_rs = self.execute_job(
+                job_id=job_id,
+                params=copy.deepcopy(context),
+            )
             context["jobs"].update(job_rs.context)
+
+            # NOTE: Mark this job queue done.
             job_queue.task_done()
 
         # NOTE: Wait for all items to finish processing
@@ -678,8 +762,12 @@ class Workflow(BaseModel):
         )
 
 
-class WorkflowSchedule(BaseModel):
-    """Workflow schedule Pydantic model."""
+class ScheduleWorkflow(BaseModel):
+    """Schedule Workflow Pydantic model that use to keep workflow model for the
+    Schedule model. it should not use Workflow model directly because on the
+    schedule config it can adjust crontab value that different from the Workflow
+    model.
+    """
 
     name: str = Field(description="A workflow name.")
     on: list[On] = Field(
@@ -692,17 +780,26 @@ class WorkflowSchedule(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare__values(cls, values: DictData) -> DictData:
-        """Prepare incoming values before validating with model fields."""
+    def __prepare_values(cls, values: DictData) -> DictData:
+        """Prepare incoming values before validating with model fields.
 
+        :rtype: DictData
+        """
         values["name"] = values["name"].replace(" ", "_")
 
         cls.__bypass_on(values)
         return values
 
     @classmethod
-    def __bypass_on(cls, data: DictData, externals: DictData | None = None):
-        """Bypass the on data to loaded config data."""
+    def __bypass_on(
+        cls,
+        data: DictData,
+        externals: DictData | None = None,
+    ) -> DictData:
+        """Bypass the on data to loaded config data.
+
+        :rtype: DictData
+        """
         if on := data.pop("on", []):
 
             if isinstance(on, str):
@@ -735,9 +832,9 @@ class Schedule(BaseModel):
             "A schedule description that can be string of markdown content."
         ),
     )
-    workflows: list[WorkflowSchedule] = Field(
+    workflows: list[ScheduleWorkflow] = Field(
         default_factory=list,
-        description="A list of WorkflowSchedule models.",
+        description="A list of ScheduleWorkflow models.",
     )
 
     @classmethod
@@ -746,6 +843,15 @@ class Schedule(BaseModel):
         name: str,
         externals: DictData | None = None,
     ) -> Self:
+        """Create Schedule instance from the Loader object that only receive
+        an input schedule name. The loader object will use this schedule name to
+        searching configuration data of this schedule model in conf path.
+
+        :param name: A schedule name that want to pass to Loader object.
+        :param externals: An external parameters that want to pass to Loader
+            object.
+        :rtype: Self
+        """
         loader: Loader = Loader(name, externals=(externals or {}))
 
         # NOTE: Validate the config type match with current connection model
@@ -766,18 +872,18 @@ class Schedule(BaseModel):
         running: dict[str, list[datetime]],
         *,
         externals: DictData | None = None,
-    ) -> list[WorkflowTask]:
+    ) -> list[WorkflowTaskData]:
         """Generate Task from the current datetime.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
         :param running: A mapping of name and list of datetime for running.
         :param externals: An external parameters that pass to the Loader object.
-        :rtype: list[WorkflowTask]
+        :rtype: list[WorkflowTaskData]
         """
 
         # NOTE: Create pair of workflow and on.
-        workflow_tasks: list[WorkflowTask] = []
+        workflow_tasks: list[WorkflowTaskData] = []
         externals: DictData = externals or {}
 
         for wfs in self.workflows:
@@ -800,7 +906,7 @@ class Schedule(BaseModel):
                 heappush(queue[wfs.name], next_running_date)
 
                 workflow_tasks.append(
-                    WorkflowTask(
+                    WorkflowTaskData(
                         workflow=wf,
                         on=on,
                         params=wfs.params,
@@ -812,12 +918,22 @@ class Schedule(BaseModel):
         return workflow_tasks
 
 
-def catch_exceptions(cancel_on_failure=False):
-    """Catch exception error from scheduler job."""
+def catch_exceptions(
+    cancel_on_failure: bool = False,
+) -> Callable[P, Optional[CancelJob]]:
+    """Catch exception error from scheduler job that running with schedule
+    package and return CancelJob if this function raise an error.
 
-    def catch_exceptions_decorator(func):
+    :param cancel_on_failure: A flag that allow to return the CancelJob or not
+        it will raise.
+    :rtype: Callable[P, Optional[CancelJob]]
+    """
 
+    def decorator(
+        func: Callable[P, Optional[CancelJob]],
+    ) -> Callable[P, Optional[CancelJob]]:
         try:
+            # NOTE: Check the function that want to handle is method or not.
             if inspect.ismethod(func):
 
                 @wraps(func)
@@ -838,11 +954,11 @@ def catch_exceptions(cancel_on_failure=False):
                 return CancelJob
             raise err
 
-    return catch_exceptions_decorator
+    return decorator
 
 
 @dataclass(frozen=True)
-class WorkflowTask:
+class WorkflowTaskData:
     """Workflow task dataclass that use to keep mapping data and objects for
     passing in multithreading task.
     """
@@ -854,19 +970,28 @@ class WorkflowTask:
     running: list[datetime] = field(compare=False, hash=False)
 
     @catch_exceptions(cancel_on_failure=True)
-    def release(self, log: Log | None = None) -> None:
+    def release(
+        self,
+        log: Log | None = None,
+        *,
+        waiting_sec: int = 60,
+        sleep_interval: int = 15,
+    ) -> None:
         """Workflow release, it will use with the same logic of
         `workflow.release` method.
 
-        :param log: A log object.
+        :param log: A log object for saving result logging from workflow
+            execution process.
+        :param waiting_sec: A second period value that allow workflow execute.
+        :param sleep_interval: A second value that want to waiting until time
+            to execute.
         """
-        tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
         log: Log = log or FileLog
         wf: Workflow = self.workflow
         on: On = self.on
 
         gen: CronRunner = on.generate(
-            datetime.now(tz=tz).replace(second=0, microsecond=0)
+            datetime.now(tz=config.tz).replace(second=0, microsecond=0)
         )
         cron_tz: ZoneInfo = gen.tz
 
@@ -883,7 +1008,7 @@ class WorkflowTask:
         )
         heappush(self.running[wf.name], next_time)
 
-        if get_diff_sec(next_time, tz=cron_tz) > 55:
+        if get_diff_sec(next_time, tz=cron_tz) > waiting_sec:
             logger.debug(
                 f"({wf.run_id}) [CORE]: {wf.name!r} : {on.cronjob} "
                 f": Does not closely >> {next_time:%Y-%m-%d %H:%M:%S}"
@@ -903,7 +1028,9 @@ class WorkflowTask:
         )
 
         # NOTE: Release when the time is nearly to schedule time.
-        while (duration := get_diff_sec(next_time, tz=tz)) > (15 + 5):
+        while (duration := get_diff_sec(next_time, tz=config.tz)) > (
+            sleep_interval + 5
+        ):
             logger.debug(
                 f"({wf.run_id}) [CORE]: {wf.name!r} : {on.cronjob} "
                 f": Sleep until: {duration}"
@@ -968,21 +1095,17 @@ class WorkflowTask:
         heappush(self.queue[wf.name], future_running_time)
         logger.debug(f"[CORE]: {'-' * 100}")
 
-    def __eq__(self, other):
-        if isinstance(other, WorkflowTask):
+    def __eq__(self, other) -> bool:
+        if isinstance(other, WorkflowTaskData):
             return (
                 self.workflow.name == other.workflow.name
                 and self.on.cronjob == other.on.cronjob
             )
 
 
-def queue2str(queue: list[datetime]) -> Iterator[str]:
-    return (f"{q:%Y-%m-%d %H:%M:%S}" for q in queue)
-
-
 @catch_exceptions(cancel_on_failure=True)
 def workflow_task(
-    workflow_tasks: list[WorkflowTask],
+    workflow_tasks: list[WorkflowTaskData],
     stop: datetime,
     threads: dict[str, Thread],
 ) -> CancelJob | None:
@@ -996,11 +1119,10 @@ def workflow_task(
     :param threads:
     :rtype: CancelJob | None
     """
-    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    start_date: datetime = datetime.now(tz=tz)
+    start_date: datetime = datetime.now(tz=config.tz)
     start_date_minute: datetime = start_date.replace(second=0, microsecond=0)
 
-    if start_date > stop.replace(tzinfo=tz):
+    if start_date > stop.replace(tzinfo=config.tz):
         logger.info("[WORKFLOW]: Stop this schedule with datetime stopper.")
         while len(threads) > 0:
             logger.warning(
@@ -1117,9 +1239,8 @@ def workflow_control(
             "Should install schedule package before use this module."
         ) from None
 
-    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
     schedule: Scheduler = Scheduler()
-    start_date: datetime = datetime.now(tz=tz)
+    start_date: datetime = datetime.now(tz=config.tz)
 
     # NOTE: Design workflow queue caching.
     #   ---
@@ -1134,7 +1255,7 @@ def workflow_control(
     )
 
     # NOTE: Create pair of workflow and on from schedule model.
-    workflow_tasks: list[WorkflowTask] = []
+    workflow_tasks: list[WorkflowTaskData] = []
     for name in schedules:
         sch: Schedule = Schedule.from_loader(name, externals=externals)
         workflow_tasks.extend(
@@ -1205,8 +1326,8 @@ def workflow_runner(
     created in config path and chuck it with WORKFLOW_APP_SCHEDULE_PER_PROCESS
     value to multiprocess executor pool.
 
-    The current workflow logic:
-    ---
+        The current workflow logic that split to process will be below diagram:
+
         PIPELINES ==> process 01 ==> schedule 1 minute --> thread of release
                                                            workflow task 01 01
                                                        --> thread of release