ddeutil-workflow 0.0.15__py3-none-any.whl → 0.0.17__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -3,13 +3,26 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""
+The main schedule running is ``workflow_runner`` function that trigger the
+multiprocess of ``workflow_control`` function for listing schedules on the
+config by ``Loader.finds(Schedule)``.
+
+    The ``workflow_control`` is the scheduler function that release 2 schedule
+functions; ``workflow_task``, and ``workflow_monitor``.
+
+    ``workflow_control`` --- Every minute at :02 --> ``workflow_task``
+                         --- Every 5 minutes     --> ``workflow_monitor``
+
+    The ``workflow_task`` will run ``task.release`` method in threading object
+for multithreading strategy. This ``release`` method will run only one crontab
+value with the on field.
+"""
 from __future__ import annotations
 
 import copy
 import inspect
-import json
 import logging
-import os
 import time
 from concurrent.futures import (
     Future,
@@ -39,18 +52,16 @@ except ImportError:
 
 try:
     from schedule import CancelJob
-except ImportError:
+except ImportError:  # pragma: no cov
     CancelJob = None
 
+from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import config
-from .cron import CronRunner
+from .conf import FileLog, Loader, Log, config, get_logger
 from .exceptions import JobException, WorkflowException
 from .job import Job
-from .log import FileLog, Log, get_logger
 from .on import On
 from .utils import (
-    Loader,
     Param,
     Result,
     batch,
@@ -75,7 +86,7 @@ __all__: TupleStr = (
     "Schedule",
     "ScheduleWorkflow",
     "workflow_task",
-    "workflow_long_running_task",
+    "workflow_monitor",
     "workflow_control",
     "workflow_runner",
 )
@@ -184,7 +195,7 @@ class Workflow(BaseModel):
         return data
 
     @model_validator(mode="before")
-    def __prepare_params(cls, values: DictData) -> DictData:
+    def __prepare_model_before__(cls, values: DictData) -> DictData:
         """Prepare the params key."""
         # NOTE: Prepare params type if it passing with only type value.
         if params := values.pop("params", {}):
@@ -199,9 +210,10 @@ class Workflow(BaseModel):
         return values
 
     @field_validator("desc", mode="after")
-    def ___prepare_desc(cls, value: str) -> str:
+    def __dedent_desc__(cls, value: str) -> str:
         """Prepare description string that was created on a template.
 
+        :param value: A description string value that want to dedent.
         :rtype: str
         """
         return dedent(value)
@@ -217,8 +229,8 @@ class Workflow(BaseModel):
                 need for need in self.jobs[job].needs if need not in self.jobs
             ]:
                 raise WorkflowException(
-                    f"This needed jobs: {not_exist} do not exist in this "
-                    f"workflow, {self.name!r}"
+                    f"The needed jobs: {not_exist} do not found in "
+                    f"{self.name!r}."
                 )
 
             # NOTE: update a job id with its job id from workflow template
@@ -341,11 +353,11 @@ class Workflow(BaseModel):
         # NOTE: get next schedule time that generate from now.
         next_time: datetime = gen.next
 
-        # NOTE: get next utils it does not logger.
+        # NOTE: While-loop to getting next until it does not logger.
         while log.is_pointed(self.name, next_time, queue=queue):
             next_time: datetime = gen.next
 
-        # NOTE: push this next running time to log queue
+        # NOTE: Heap-push this next running time to log queue list.
         heappush(queue, next_time)
 
         # VALIDATE: Check the different time between the next schedule time and
@@ -458,8 +470,10 @@ class Workflow(BaseModel):
         queue: list[datetime] = []
         results: list[Result] = []
 
-        worker: int = int(os.getenv("WORKFLOW_CORE_MAX_NUM_POKING") or "4")
-        with ThreadPoolExecutor(max_workers=worker) as executor:
+        with ThreadPoolExecutor(
+            max_workers=config.max_poking_pool_worker,
+            thread_name_prefix="wf_poking_",
+        ) as executor:
             futures: list[Future] = []
             for on in self.on:
                 futures.append(
@@ -694,7 +708,7 @@ class Workflow(BaseModel):
                     raise WorkflowException(f"{err}")
                 try:
                     future.result(timeout=60)
-                except TimeoutError as err:
+                except TimeoutError as err:  # pragma: no cove
                     raise WorkflowException(
                         "Timeout when getting result from future"
                     ) from err
@@ -795,7 +809,7 @@ class ScheduleWorkflow(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values: DictData) -> DictData:
+    def __prepare_before__(cls, values: DictData) -> DictData:
         """Prepare incoming values before validating with model fields.
 
         :rtype: DictData
@@ -933,9 +947,11 @@ class Schedule(BaseModel):
         return workflow_tasks
 
 
-def catch_exceptions(
-    cancel_on_failure: bool = False,
-) -> Callable[P, Optional[CancelJob]]:
+ReturnCancelJob = Callable[P, Optional[CancelJob]]
+DecoratorCancelJob = Callable[[ReturnCancelJob], ReturnCancelJob]
+
+
+def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
     """Catch exception error from scheduler job that running with schedule
     package and return CancelJob if this function raise an error.
 
@@ -944,9 +960,7 @@ def catch_exceptions(
     :rtype: Callable[P, Optional[CancelJob]]
     """
 
-    def decorator(
-        func: Callable[P, Optional[CancelJob]],
-    ) -> Callable[P, Optional[CancelJob]]:
+    def decorator(func: ReturnCancelJob) -> ReturnCancelJob:
         try:
             # NOTE: Check the function that want to handle is method or not.
             if inspect.ismethod(func):
@@ -981,8 +995,8 @@ class WorkflowTaskData:
     workflow: Workflow
     on: On
     params: DictData = field(compare=False, hash=False)
-    queue: list[datetime] = field(compare=False, hash=False)
-    running: list[datetime] = field(compare=False, hash=False)
+    queue: dict[str, list[datetime]] = field(compare=False, hash=False)
+    running: dict[str, list[datetime]] = field(compare=False, hash=False)
 
     @catch_exceptions(cancel_on_failure=True)
     def release(
@@ -1062,8 +1076,9 @@ class WorkflowTaskData:
             },
         }
 
-        # WARNING: Re-create workflow object that use new running workflow
-        #   ID.
+        # WARNING:
+        #   Re-create workflow object that use new running workflow ID.
+        #
         runner: Workflow = wf.get_running_id(run_id=wf.new_run_id)
         rs: Result = runner.execute(
             params=param2template(self.params, release_params),
@@ -1116,6 +1131,7 @@ class WorkflowTaskData:
                 self.workflow.name == other.workflow.name
                 and self.on.cronjob == other.on.cronjob
             )
+        return NotImplemented
 
 
 @catch_exceptions(cancel_on_failure=True)
@@ -1127,10 +1143,10 @@ def workflow_task(
     """Workflow task generator that create release pair of workflow and on to
     the threading in background.
 
-        This workflow task will start every minute at :02 second.
+        This workflow task will start every minute at ':02' second.
 
     :param workflow_tasks:
-    :param stop:
+    :param stop: A stop datetime object that force stop running scheduler.
     :param threads:
     :rtype: CancelJob | None
     """
@@ -1145,7 +1161,7 @@ def workflow_task(
                 "running in background."
             )
             time.sleep(15)
-            workflow_long_running_task(threads)
+            workflow_monitor(threads)
         return CancelJob
 
     # IMPORTANT:
@@ -1217,7 +1233,7 @@ def workflow_task(
     logger.debug(f"[WORKFLOW]: {'=' * 100}")
 
 
-def workflow_long_running_task(threads: dict[str, Thread]) -> None:
+def workflow_monitor(threads: dict[str, Thread]) -> None:
     """Workflow schedule for monitoring long running thread from the schedule
     control.
 
@@ -1275,30 +1291,29 @@ def workflow_control(
         sch: Schedule = Schedule.from_loader(name, externals=externals)
         workflow_tasks.extend(
             sch.tasks(
-                start_date_waiting, wf_queue, wf_running, externals=externals
+                start_date_waiting,
+                queue=wf_queue,
+                running=wf_running,
+                externals=externals,
             ),
         )
 
     # NOTE: This schedule job will start every minute at :02 seconds.
-    schedule.every(1).minutes.at(":02").do(
-        workflow_task,
-        workflow_tasks=workflow_tasks,
-        stop=stop
-        or (
-            start_date
-            + timedelta(
-                **json.loads(
-                    os.getenv("WORKFLOW_APP_STOP_BOUNDARY_DELTA")
-                    or '{"minutes": 5, "seconds": 20}'
-                )
-            )
-        ),
-        threads=thread_releases,
-    ).tag("control")
+    (
+        schedule.every(1)
+        .minutes.at(":02")
+        .do(
+            workflow_task,
+            workflow_tasks=workflow_tasks,
+            stop=(stop or (start_date + config.stop_boundary_delta)),
+            threads=thread_releases,
+        )
+        .tag("control")
+    )
 
     # NOTE: Checking zombie task with schedule job will start every 5 minute.
     schedule.every(5).minutes.at(":10").do(
-        workflow_long_running_task,
+        workflow_monitor,
         threads=thread_releases,
     ).tag("monitor")
 
@@ -1332,14 +1347,16 @@ def workflow_runner(
     """Workflow application that running multiprocessing schedule with chunk of
     workflows that exists in config path.
 
-    :param stop:
+    :param stop: A stop datetime object that force stop running scheduler.
     :param excluded:
     :param externals:
+
     :rtype: list[str]
 
         This function will get all workflows that include on value that was
-    created in config path and chuck it with WORKFLOW_APP_SCHEDULE_PER_PROCESS
-    value to multiprocess executor pool.
+    created in config path and chuck it with application config variable
+    ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS`` env var to multiprocess executor
+    pool.
 
         The current workflow logic that split to process will be below diagram:
 
@@ -1356,7 +1373,7 @@ def workflow_runner(
     excluded: list[str] = excluded or []
 
     with ProcessPoolExecutor(
-        max_workers=int(os.getenv("WORKFLOW_APP_PROCESS_WORKER") or "2"),
+        max_workers=config.max_schedule_process,
     ) as executor:
         futures: list[Future] = [
             executor.submit(
@@ -1367,7 +1384,7 @@ def workflow_runner(
             )
             for loader in batch(
                 Loader.finds(Schedule, excluded=excluded),
-                n=int(os.getenv("WORKFLOW_APP_SCHEDULE_PER_PROCESS") or "100"),
+                n=config.max_schedule_per_process,
             )
         ]
 

ddeutil/workflow/stage.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Stage Model that use for getting stage data template from Job Model.
-The stage that handle the minimize task that run in some thread (same thread at
+"""Stage Model that use for getting stage data template from the Job Model.
+The stage handle the minimize task that run in some thread (same thread at
 its job owner) that mean it is the lowest executor of a workflow workflow that
 can tracking logs.
 
@@ -12,11 +12,13 @@ can tracking logs.
 handle stage error on this stage model. I think stage model should have a lot of
 usecase and it does not worry when I want to create a new one.
 
-    Execution   --> Ok    --> Result with 0
-                --> Error --> Raise StageException
+    Execution   --> Ok      --> Result with 0
+                --> Error   --> Result with 1 (if env var was set)
+                            --> Raise StageException
 
-    On the context I/O that pass to stage object at execute process. The execute
-method receive `{"params": {...}}` for mapping to template.
+    On the context I/O that pass to a stage object at execute process. The
+execute method receives a `params={"params": {...}}` value for mapping to
+template searching.
 """
 from __future__ import annotations
 
@@ -46,9 +48,8 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Re, TupleStr
-from .conf import config
+from .conf import config, get_logger
 from .exceptions import StageException
-from .log import get_logger
 from .utils import (
     Registry,
     Result,
@@ -88,20 +89,28 @@ def handler_result(message: str | None = None) -> DecoratorResult:
     you force catching an output result with error message by specific
     environment variable,`WORKFLOW_CORE_STAGE_RAISE_ERROR`.
 
-        Execution   --> Ok      --> Result with 0
+        Execution   --> Ok      --> Result
+                                        status: 0
+                                        context:
+                                            outputs: ...
+                    --> Error   --> Result (if env var was set)
+                                        status: 1
+                                        context:
+                                            error: ...
+                                            error_message: ...
                     --> Error   --> Raise StageException
-                                --> Result with 1 (if env var was set)
 
         On the last step, it will set the running ID on a return result object
     from current stage ID before release the final result.
 
     :param message: A message that want to add at prefix of exception statement.
+    :type message: str | None (Default=None)
     :rtype: Callable[P, Result]
     """
     # NOTE: The prefix message string that want to add on the first exception
     #   message dialog.
     #
-    #       ... ValueError: {message}
+    #       >>> ValueError: {message}
     #       ...     raise value error from the stage execution process.
     #
     message: str = message or ""
@@ -118,6 +127,7 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
+                print("Stage Raise error:", config.stage_raise_error)
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
                     #   itself, it will return that error with previous
@@ -175,11 +185,14 @@ class BaseStage(BaseModel, ABC):
     )
 
     @model_validator(mode="after")
-    def __prepare_running_id(self) -> Self:
+    def __prepare_running_id__(self) -> Self:
         """Prepare stage running ID that use default value of field and this
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
 
+        :raise ValueError: When the ID and name fields include matrix parameter
+            template with the 'matrix.' string value.
+
         :rtype: Self
         """
         if self.run_id is None:
@@ -189,7 +202,7 @@ class BaseStage(BaseModel, ABC):
         #   template. (allow only matrix)
         if not_in_template(self.id) or not_in_template(self.name):
             raise ValueError(
-                "Stage name and ID should only template with matrix."
+                "Stage name and ID should only template with 'matrix.'"
             )
 
         return self
@@ -226,16 +239,16 @@ class BaseStage(BaseModel, ABC):
         The result of the `to` variable will be;
 
             ... (iii) to: {
-                            'stages': {
-                                '<stage-id>': {'outputs': {'foo': 'bar'}}
-                            }
+                        'stages': {
+                            '<stage-id>': {'outputs': {'foo': 'bar'}}
                         }
+                    }
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
-        if not (self.id or config.stage_default_id):
+        if self.id is None and not config.stage_default_id:
             logger.debug(
                 f"({self.run_id}) [STAGE]: Output does not set because this "
                 f"stage does not set ID or default stage ID config flag not be "
@@ -255,7 +268,7 @@ class BaseStage(BaseModel, ABC):
         )
 
         # NOTE: Set the output to that stage generated ID with ``outputs`` key.
-        logger.debug(f"({self.run_id}) [STAGE]: Set outputs on: {_id}")
+        logger.debug(f"({self.run_id}) [STAGE]: Set outputs to {_id!r}")
         to["stages"][_id] = {"outputs": output}
         return to
 
@@ -263,6 +276,11 @@ class BaseStage(BaseModel, ABC):
         """Return true if condition of this stage do not correct. This process
         use build-in eval function to execute the if-condition.
 
+        :raise StageException: When it has any error raise from the eval
+            condition statement.
+        :raise StageException: When return type of the eval condition statement
+            does not return with boolean type.
+
         :param params: A parameters that want to pass to condition template.
         :rtype: bool
         """
@@ -299,6 +317,7 @@ class EmptyStage(BaseStage):
     sleep: float = Field(
         default=0,
         description="A second value to sleep before finish execution",
+        ge=0,
     )
 
     def execute(self, params: DictData) -> Result:
@@ -351,7 +370,7 @@ class BashStage(BaseStage):
     )
 
     @contextlib.contextmanager
-    def __prepare_bash(self, bash: str, env: DictStr) -> Iterator[TupleStr]:
+    def prepare_bash(self, bash: str, env: DictStr) -> Iterator[TupleStr]:
         """Return context of prepared bash statement that want to execute. This
         step will write the `.sh` file before giving this file name to context.
         After that, it will auto delete this file automatic.
@@ -394,15 +413,12 @@ class BashStage(BaseStage):
         :rtype: Result
         """
         bash: str = param2template(dedent(self.bash), params)
-        with self.__prepare_bash(
+        with self.prepare_bash(
             bash=bash, env=param2template(self.env, params)
         ) as sh:
             logger.info(f"({self.run_id}) [STAGE]: Shell-Execute: {sh}")
             rs: CompletedProcess = subprocess.run(
-                sh,
-                shell=False,
-                capture_output=True,
-                text=True,
+                sh, shell=False, capture_output=True, text=True
             )
         if rs.returncode > 0:
             # NOTE: Prepare stderr message that returning from subprocess.
@@ -419,8 +435,8 @@ class BashStage(BaseStage):
             status=0,
             context={
                 "return_code": rs.returncode,
-                "stdout": rs.stdout.rstrip("\n"),
-                "stderr": rs.stderr.rstrip("\n"),
+                "stdout": rs.stdout.rstrip("\n") or None,
+                "stderr": rs.stderr.rstrip("\n") or None,
             },
         )
 
@@ -452,6 +468,15 @@ class PyStage(BaseStage):
         ),
     )
 
+    @staticmethod
+    def pick_keys_from_locals(values: DictData) -> Iterator[str]:
+        from inspect import ismodule
+
+        for value in values:
+            if value == "__annotations__" or ismodule(values[value]):
+                continue
+            yield value
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Override set an outputs method for the Python execution process that
         extract output from all the locals values.
@@ -461,15 +486,19 @@ class PyStage(BaseStage):
         :rtype: DictData
         """
         # NOTE: The output will fileter unnecessary keys from locals.
-        _locals: DictData = output["locals"]
+        lc: DictData = output.get("locals", {})
         super().set_outputs(
-            {k: _locals[k] for k in _locals if k != "__annotations__"}, to=to
+            (
+                {k: lc[k] for k in self.pick_keys_from_locals(lc)}
+                | {k: output[k] for k in output if k.startswith("error")}
+            ),
+            to=to,
         )
 
-        # NOTE:
-        #   Override value that changing from the globals that pass via exec.
-        _globals: DictData = output["globals"]
-        to.update({k: _globals[k] for k in to if k in _globals})
+        # NOTE: Override value that changing from the globals that pass via the
+        #   exec function.
+        gb: DictData = output.get("globals", {})
+        to.update({k: gb[k] for k in to if k in gb})
         return to
 
     @handler_result()
@@ -487,15 +516,15 @@ class PyStage(BaseStage):
         _globals: DictData = (
             globals() | params | param2template(self.vars, params)
         )
-        _locals: DictData = {}
+        lc: DictData = {}
 
         # NOTE: Start exec the run statement.
         logger.info(f"({self.run_id}) [STAGE]: Py-Execute: {self.name}")
-        exec(run, _globals, _locals)
+        exec(run, _globals, lc)
 
         return Result(
             status=0,
-            context={"locals": _locals, "globals": _globals},
+            context={"locals": lc, "globals": _globals},
         )
 
 
@@ -514,6 +543,11 @@ def extract_hook(hook: str) -> Callable[[], TagFunc]:
     """Extract Hook function from string value to hook partial function that
     does run it at runtime.
 
+    :raise NotImplementedError: When the searching hook's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching hook's tag result does not
+        exists in the registry with its function key.
+
     :param hook: A hook value that able to match with Task regex.
     :rtype: Callable[[], TagFunc]
     """
@@ -554,14 +588,14 @@ class HookStage(BaseStage):
         >>> stage = {
         ...     "name": "Task stage execution",
         ...     "uses": "tasks/function-name@tag-name",
-        ...     "args": {
-        ...         "FOO": "BAR",
-        ...     },
+        ...     "args": {"FOO": "BAR"},
         ... }
     """
 
     uses: str = Field(
-        description="A pointer that want to load function from registry.",
+        description=(
+            "A pointer that want to load function from the hook registry."
+        ),
     )
     args: DictData = Field(
         default_factory=dict,
@@ -573,6 +607,11 @@ class HookStage(BaseStage):
     def execute(self, params: DictData) -> Result:
         """Execute the Hook function that already in the hook registry.
 
+        :raise ValueError: When the necessary arguments of hook function do not
+            set from the input params argument.
+        :raise TypeError: When the return type of hook function does not be
+            dict type.
+
         :param params: A parameter that want to pass before run any statement.
         :type params: DictData
         :rtype: Result
@@ -622,10 +661,7 @@ class TriggerStage(BaseStage):
         >>> stage = {
         ...     "name": "Trigger workflow stage execution",
         ...     "trigger": 'workflow-name-for-loader',
-        ...     "params": {
-        ...         "run-date": "2024-08-01",
-        ...         "source": "src",
-        ...     },
+        ...     "params": {"run-date": "2024-08-01", "source": "src"},
         ... }
     """