ddeutil-workflow 0.0.15__py3-none-any.whl → 0.0.16__py3-none-any.whl

ddeutil/workflow/repeat.py

@@ -6,14 +6,13 @@
 from __future__ import annotations
 
 import asyncio
-import os
 from asyncio import ensure_future
 from datetime import datetime
 from functools import wraps
-from zoneinfo import ZoneInfo
 
 from starlette.concurrency import run_in_threadpool
 
+from .conf import config
 from .cron import CronJob
 from .log import get_logger
 
@@ -24,9 +23,7 @@ def get_cronjob_delta(cron: str) -> float:
     """This function returns the time delta between now and the next cron
     execution time.
     """
-    now: datetime = datetime.now(
-        tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    )
+    now: datetime = datetime.now(tz=config.tz)
     cron = CronJob(cron)
     return (cron.schedule(now).next - now).total_seconds()
 

ddeutil/workflow/route.py

@@ -6,10 +6,8 @@
 from __future__ import annotations
 
 import copy
-import os
 from datetime import datetime, timedelta
 from typing import Any
-from zoneinfo import ZoneInfo
 
 from fastapi import APIRouter, HTTPException, Request
 from fastapi import status as st
@@ -18,9 +16,10 @@ from pydantic import BaseModel
 
 from . import Workflow
 from .__types import DictData
+from .conf import Loader, config
 from .log import get_logger
 from .scheduler import Schedule
-from .utils import Loader, Result
+from .utils import Result
 
 logger = get_logger("ddeutil.workflow")
 workflow = APIRouter(
@@ -87,12 +86,7 @@ async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
     # NOTE: Start execute manually
     rs: Result = wf.execute(params=payload.params)
 
-    return rs.model_dump(
-        by_alias=True,
-        exclude_none=True,
-        exclude_unset=True,
-        exclude_defaults=True,
-    )
+    return dict(rs)
 
 
 @workflow.get("/{name}/logs")
@@ -172,8 +166,7 @@ async def add_deploy_scheduler(request: Request, name: str):
 
     request.state.scheduler.append(name)
 
-    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    start_date: datetime = datetime.now(tz=tz)
+    start_date: datetime = datetime.now(tz=config.tz)
     start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
         second=0, microsecond=0
     )

ddeutil/workflow/scheduler.py

@@ -3,13 +3,26 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""
+The main schedule running is ``workflow_runner`` function that trigger the
+multiprocess of ``workflow_control`` function for listing schedules on the
+config by ``Loader.finds(Schedule)``.
+
+    The ``workflow_control`` is the scheduler function that release 2 schedule
+functions; ``workflow_task``, and ``workflow_monitor``.
+
+    ``workflow_control`` --- Every minute at :02 --> ``workflow_task``
+                         --- Every 5 minutes     --> ``workflow_monitor``
+
+    The ``workflow_task`` will run ``task.release`` method in threading object
+for multithreading strategy. This ``release`` method will run only one crontab
+value with the on field.
+"""
 from __future__ import annotations
 
 import copy
 import inspect
-import json
 import logging
-import os
 import time
 from concurrent.futures import (
     Future,
@@ -43,14 +56,13 @@ except ImportError:
     CancelJob = None
 
 from .__types import DictData, TupleStr
-from .conf import config
+from .conf import Loader, config
 from .cron import CronRunner
 from .exceptions import JobException, WorkflowException
 from .job import Job
 from .log import FileLog, Log, get_logger
 from .on import On
 from .utils import (
-    Loader,
     Param,
     Result,
     batch,
@@ -75,7 +87,7 @@ __all__: TupleStr = (
     "Schedule",
     "ScheduleWorkflow",
     "workflow_task",
-    "workflow_long_running_task",
+    "workflow_monitor",
     "workflow_control",
     "workflow_runner",
 )
@@ -184,7 +196,7 @@ class Workflow(BaseModel):
         return data
 
     @model_validator(mode="before")
-    def __prepare_params(cls, values: DictData) -> DictData:
+    def __prepare_model_before__(cls, values: DictData) -> DictData:
         """Prepare the params key."""
         # NOTE: Prepare params type if it passing with only type value.
         if params := values.pop("params", {}):
@@ -199,9 +211,10 @@ class Workflow(BaseModel):
         return values
 
     @field_validator("desc", mode="after")
-    def ___prepare_desc(cls, value: str) -> str:
+    def __dedent_desc__(cls, value: str) -> str:
         """Prepare description string that was created on a template.
 
+        :param value: A description string value that want to dedent.
         :rtype: str
         """
         return dedent(value)
@@ -458,8 +471,10 @@ class Workflow(BaseModel):
         queue: list[datetime] = []
         results: list[Result] = []
 
-        worker: int = int(os.getenv("WORKFLOW_CORE_MAX_NUM_POKING") or "4")
-        with ThreadPoolExecutor(max_workers=worker) as executor:
+        with ThreadPoolExecutor(
+            max_workers=config.max_poking_pool_worker,
+            thread_name_prefix="wf_poking_",
+        ) as executor:
             futures: list[Future] = []
             for on in self.on:
                 futures.append(
@@ -795,7 +810,7 @@ class ScheduleWorkflow(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values: DictData) -> DictData:
+    def __prepare_before__(cls, values: DictData) -> DictData:
         """Prepare incoming values before validating with model fields.
 
         :rtype: DictData
@@ -933,9 +948,11 @@ class Schedule(BaseModel):
         return workflow_tasks
 
 
-def catch_exceptions(
-    cancel_on_failure: bool = False,
-) -> Callable[P, Optional[CancelJob]]:
+ReturnCancelJob = Callable[P, Optional[CancelJob]]
+DecoratorCancelJob = Callable[[ReturnCancelJob], ReturnCancelJob]
+
+
+def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
     """Catch exception error from scheduler job that running with schedule
     package and return CancelJob if this function raise an error.
 
@@ -944,9 +961,7 @@ def catch_exceptions(
     :rtype: Callable[P, Optional[CancelJob]]
     """
 
-    def decorator(
-        func: Callable[P, Optional[CancelJob]],
-    ) -> Callable[P, Optional[CancelJob]]:
+    def decorator(func: ReturnCancelJob) -> ReturnCancelJob:
         try:
             # NOTE: Check the function that want to handle is method or not.
             if inspect.ismethod(func):
@@ -981,8 +996,8 @@ class WorkflowTaskData:
     workflow: Workflow
     on: On
     params: DictData = field(compare=False, hash=False)
-    queue: list[datetime] = field(compare=False, hash=False)
-    running: list[datetime] = field(compare=False, hash=False)
+    queue: dict[str, list[datetime]] = field(compare=False, hash=False)
+    running: dict[str, list[datetime]] = field(compare=False, hash=False)
 
     @catch_exceptions(cancel_on_failure=True)
     def release(
@@ -1062,8 +1077,9 @@ class WorkflowTaskData:
             },
         }
 
-        # WARNING: Re-create workflow object that use new running workflow
-        #   ID.
+        # WARNING:
+        #   Re-create workflow object that use new running workflow ID.
+        #
         runner: Workflow = wf.get_running_id(run_id=wf.new_run_id)
         rs: Result = runner.execute(
             params=param2template(self.params, release_params),
@@ -1116,6 +1132,7 @@ class WorkflowTaskData:
                 self.workflow.name == other.workflow.name
                 and self.on.cronjob == other.on.cronjob
             )
+        return NotImplemented
 
 
 @catch_exceptions(cancel_on_failure=True)
@@ -1127,10 +1144,10 @@ def workflow_task(
     """Workflow task generator that create release pair of workflow and on to
     the threading in background.
 
-        This workflow task will start every minute at :02 second.
+        This workflow task will start every minute at ':02' second.
 
     :param workflow_tasks:
-    :param stop:
+    :param stop: A stop datetime object that force stop running scheduler.
     :param threads:
     :rtype: CancelJob | None
     """
@@ -1145,7 +1162,7 @@ def workflow_task(
                 "running in background."
             )
             time.sleep(15)
-            workflow_long_running_task(threads)
+            workflow_monitor(threads)
         return CancelJob
 
     # IMPORTANT:
@@ -1217,7 +1234,7 @@ def workflow_task(
     logger.debug(f"[WORKFLOW]: {'=' * 100}")
 
 
-def workflow_long_running_task(threads: dict[str, Thread]) -> None:
+def workflow_monitor(threads: dict[str, Thread]) -> None:
     """Workflow schedule for monitoring long running thread from the schedule
     control.
 
@@ -1275,30 +1292,29 @@ def workflow_control(
         sch: Schedule = Schedule.from_loader(name, externals=externals)
         workflow_tasks.extend(
             sch.tasks(
-                start_date_waiting, wf_queue, wf_running, externals=externals
+                start_date_waiting,
+                queue=wf_queue,
+                running=wf_running,
+                externals=externals,
             ),
         )
 
     # NOTE: This schedule job will start every minute at :02 seconds.
-    schedule.every(1).minutes.at(":02").do(
-        workflow_task,
-        workflow_tasks=workflow_tasks,
-        stop=stop
-        or (
-            start_date
-            + timedelta(
-                **json.loads(
-                    os.getenv("WORKFLOW_APP_STOP_BOUNDARY_DELTA")
-                    or '{"minutes": 5, "seconds": 20}'
-                )
-            )
-        ),
-        threads=thread_releases,
-    ).tag("control")
+    (
+        schedule.every(1)
+        .minutes.at(":02")
+        .do(
+            workflow_task,
+            workflow_tasks=workflow_tasks,
+            stop=(stop or (start_date + config.stop_boundary_delta)),
+            threads=thread_releases,
+        )
+        .tag("control")
+    )
 
     # NOTE: Checking zombie task with schedule job will start every 5 minute.
     schedule.every(5).minutes.at(":10").do(
-        workflow_long_running_task,
+        workflow_monitor,
         threads=thread_releases,
     ).tag("monitor")
 
@@ -1332,14 +1348,16 @@ def workflow_runner(
     """Workflow application that running multiprocessing schedule with chunk of
     workflows that exists in config path.
 
-    :param stop:
+    :param stop: A stop datetime object that force stop running scheduler.
     :param excluded:
     :param externals:
+
     :rtype: list[str]
 
         This function will get all workflows that include on value that was
-    created in config path and chuck it with WORKFLOW_APP_SCHEDULE_PER_PROCESS
-    value to multiprocess executor pool.
+    created in config path and chuck it with application config variable
+    ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS`` env var to multiprocess executor
+    pool.
 
         The current workflow logic that split to process will be below diagram:
 
@@ -1356,7 +1374,7 @@ def workflow_runner(
     excluded: list[str] = excluded or []
 
     with ProcessPoolExecutor(
-        max_workers=int(os.getenv("WORKFLOW_APP_PROCESS_WORKER") or "2"),
+        max_workers=config.max_schedule_process,
     ) as executor:
         futures: list[Future] = [
             executor.submit(
@@ -1367,7 +1385,7 @@ def workflow_runner(
             )
             for loader in batch(
                 Loader.finds(Schedule, excluded=excluded),
-                n=int(os.getenv("WORKFLOW_APP_SCHEDULE_PER_PROCESS") or "100"),
+                n=config.max_schedule_per_process,
             )
         ]
 

ddeutil/workflow/stage.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Stage Model that use for getting stage data template from Job Model.
-The stage that handle the minimize task that run in some thread (same thread at
+"""Stage Model that use for getting stage data template from the Job Model.
+The stage handle the minimize task that run in some thread (same thread at
 its job owner) that mean it is the lowest executor of a workflow workflow that
 can tracking logs.
 
@@ -12,11 +12,13 @@ can tracking logs.
 handle stage error on this stage model. I think stage model should have a lot of
 usecase and it does not worry when I want to create a new one.
 
-    Execution   --> Ok    --> Result with 0
-                --> Error --> Raise StageException
+    Execution   --> Ok      --> Result with 0
+                --> Error   --> Raise StageException
+                            --> Result with 1 (if env var was set)
 
-    On the context I/O that pass to stage object at execute process. The execute
-method receive `{"params": {...}}` for mapping to template.
+    On the context I/O that pass to a stage object at execute process. The
+execute method receives a `params={"params": {...}}` value for mapping to
+template searching.
 """
 from __future__ import annotations
 
@@ -88,20 +90,28 @@ def handler_result(message: str | None = None) -> DecoratorResult:
     you force catching an output result with error message by specific
     environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
 
-        Execution   --> Ok      --> Result with 0
+        Execution   --> Ok      --> Result
+                                        status: 0
+                                        context:
+                                            outputs: ...
                     --> Error   --> Raise StageException
-                                --> Result with 1 (if env var was set)
+                                --> Result (if env var was set)
+                                        status: 1
+                                        context:
+                                            error: ...
+                                            error_message: ...
 
         On the last step, it will set the running ID on a return result object
     from current stage ID before release the final result.
 
     :param message: A message that want to add at prefix of exception statement.
+    :type message: str | None (Default=None)
     :rtype: Callable[P, Result]
     """
     # NOTE: The prefix message string that want to add on the first exception
     #   message dialog.
     #
-    #       ... ValueError: {message}
+    #       >>> ValueError: {message}
     #       ...     raise value error from the stage execution process.
     #
     message: str = message or ""
@@ -175,7 +185,7 @@ class BaseStage(BaseModel, ABC):
     )
 
     @model_validator(mode="after")
-    def __prepare_running_id(self) -> Self:
+    def __prepare_running_id__(self) -> Self:
         """Prepare stage running ID that use default value of field and this
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
@@ -235,7 +245,7 @@ class BaseStage(BaseModel, ABC):
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
-        if not (self.id or config.stage_default_id):
+        if self.id is None and not config.stage_default_id:
             logger.debug(
                 f"({self.run_id}) [STAGE]: Output does not set because this "
                 f"stage does not set ID or default stage ID config flag not be "
@@ -255,7 +265,7 @@ class BaseStage(BaseModel, ABC):
         )
 
         # NOTE: Set the output to that stage generated ID with ``outputs`` key.
-        logger.debug(f"({self.run_id}) [STAGE]: Set outputs on: {_id}")
+        logger.debug(f"({self.run_id}) [STAGE]: Set outputs to {_id!r}")
         to["stages"][_id] = {"outputs": output}
         return to
 
@@ -299,6 +309,7 @@ class EmptyStage(BaseStage):
     sleep: float = Field(
         default=0,
         description="A second value to sleep before finish execution",
+        ge=0,
     )
 
     def execute(self, params: DictData) -> Result:
@@ -351,7 +362,7 @@ class BashStage(BaseStage):
     )
 
     @contextlib.contextmanager
-    def __prepare_bash(self, bash: str, env: DictStr) -> Iterator[TupleStr]:
+    def prepare_bash(self, bash: str, env: DictStr) -> Iterator[TupleStr]:
         """Return context of prepared bash statement that want to execute. This
         step will write the `.sh` file before giving this file name to context.
         After that, it will auto delete this file automatic.
@@ -394,15 +405,12 @@ class BashStage(BaseStage):
         :rtype: Result
         """
         bash: str = param2template(dedent(self.bash), params)
-        with self.__prepare_bash(
+        with self.prepare_bash(
             bash=bash, env=param2template(self.env, params)
         ) as sh:
             logger.info(f"({self.run_id}) [STAGE]: Shell-Execute: {sh}")
             rs: CompletedProcess = subprocess.run(
-                sh,
-                shell=False,
-                capture_output=True,
-                text=True,
+                sh, shell=False, capture_output=True, text=True
             )
         if rs.returncode > 0:
             # NOTE: Prepare stderr message that returning from subprocess.
@@ -419,8 +427,8 @@ class BashStage(BaseStage):
             status=0,
             context={
                 "return_code": rs.returncode,
-                "stdout": rs.stdout.rstrip("\n"),
-                "stderr": rs.stderr.rstrip("\n"),
+                "stdout": rs.stdout.rstrip("\n") or None,
+                "stderr": rs.stderr.rstrip("\n") or None,
             },
         )
 
@@ -554,14 +562,14 @@ class HookStage(BaseStage):
         >>> stage = {
         ...     "name": "Task stage execution",
         ...     "uses": "tasks/function-name@tag-name",
-        ...     "args": {
-        ...         "FOO": "BAR",
-        ...     },
+        ...     "args": {"FOO": "BAR"},
         ... }
     """
 
     uses: str = Field(
-        description="A pointer that want to load function from registry.",
+        description=(
+            "A pointer that want to load function from the hook registry."
+        ),
     )
     args: DictData = Field(
         default_factory=dict,
@@ -622,10 +630,7 @@ class TriggerStage(BaseStage):
         >>> stage = {
         ...     "name": "Trigger workflow stage execution",
         ...     "trigger": 'workflow-name-for-loader',
-        ...     "params": {
-        ...         "run-date": "2024-08-01",
-        ...         "source": "src",
-        ...     },
+        ...     "params": {"run-date": "2024-08-01", "source": "src"},
         ... }
     """