ddeutil-workflow 0.0.31__py3-none-any.whl → 0.0.33__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -51,7 +51,8 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, Log, config, get_log, get_logger
+from .audit import Audit, get_audit
+from .conf import Loader, config, get_logger
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
 from .result import Result
@@ -275,37 +276,6 @@ class Schedule(BaseModel):
 
         return cls.model_validate(obj=loader_data)
 
-    @classmethod
-    def extract_tasks(
-        cls,
-        schedules: list[str],
-        start_date: datetime,
-        queue: dict[str, ReleaseQueue],
-        externals: DictData | None = None,
-    ) -> list[WorkflowTask]:
-        """Return the list of WorkflowTask object from all schedule object that
-        include in an input schedules argument.
-
-        :param schedules: A list of schedule name that will use `from_loader`
-            method.
-        :param start_date: A start date that get from the workflow schedule.
-        :param queue: A mapping of name and list of datetime for queue.
-        :param externals: An external parameters that pass to the Loader object.
-
-        :rtype: list[WorkflowTask]
-        """
-        tasks: list[WorkflowTask] = []
-        for name in schedules:
-            schedule: Schedule = Schedule.from_loader(name, externals=externals)
-            tasks.extend(
-                schedule.tasks(
-                    start_date,
-                    queue=queue,
-                    externals=externals,
-                ),
-            )
-        return tasks
-
     def tasks(
         self,
         start_date: datetime,
@@ -339,6 +309,99 @@ class Schedule(BaseModel):
 
         return workflow_tasks
 
+    def pending(
+        self,
+        *,
+        stop: datetime | None = None,
+        externals: DictData | None = None,
+        log: type[Audit] | None = None,
+    ) -> None:  # pragma: no cov
+        """Pending this schedule tasks with the schedule package.
+
+        :param stop: A datetime value that use to stop running schedule.
+        :param externals: An external parameters that pass to Loader.
+        :param log: A log class that use on the workflow task release for
+            writing its release log context.
+        """
+        try:
+            from schedule import Scheduler
+        except ImportError:
+            raise ImportError(
+                "Should install schedule package before use this method."
+            ) from None
+
+        # NOTE: Get default logging.
+        log: type[Audit] = log or get_audit()
+        scheduler: Scheduler = Scheduler()
+
+        # NOTE: Create the start and stop datetime.
+        start_date: datetime = datetime.now(tz=config.tz)
+        stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+
+        # IMPORTANT: Create main mapping of queue and thread object.
+        queue: dict[str, ReleaseQueue] = {}
+        threads: ReleaseThreads = {}
+
+        start_date_waiting: datetime = start_date.replace(
+            second=0, microsecond=0
+        ) + timedelta(minutes=1)
+
+        # NOTE: This schedule job will start every minute at :02 seconds.
+        (
+            scheduler.every(1)
+            .minutes.at(":02")
+            .do(
+                schedule_task,
+                tasks=self.tasks(
+                    start_date_waiting, queue=queue, externals=externals
+                ),
+                stop=stop_date,
+                queue=queue,
+                threads=threads,
+                log=log,
+            )
+            .tag("control")
+        )
+
+        # NOTE: Checking zombie task with schedule job will start every 5 minute at
+        #   :10 seconds.
+        (
+            scheduler.every(5)
+            .minutes.at(":10")
+            .do(
+                monitor,
+                threads=threads,
+            )
+            .tag("monitor")
+        )
+
+        # NOTE: Start running schedule
+        logger.info(
+            f"[SCHEDULE]: Schedule with stopper: {stop_date:%Y-%m-%d %H:%M:%S}"
+        )
+
+        while True:
+            scheduler.run_pending()
+            time.sleep(1)
+
+            # NOTE: Break the scheduler when the control job does not exist.
+            if not scheduler.get_jobs("control"):
+                scheduler.clear("monitor")
+
+                while len(threads) > 0:
+                    logger.warning(
+                        "[SCHEDULE]: Waiting schedule release thread that still "
+                        "running in background."
+                    )
+                    delay(10)
+                    monitor(threads)
+
+                break
+
+        logger.warning(
+            f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
+        )
+
 
 ResultOrCancelJob = Union[type[CancelJob], Result]
 ReturnCancelJob = Callable[P, ResultOrCancelJob]
@@ -388,13 +451,13 @@ def schedule_task(
     stop: datetime,
     queue: dict[str, ReleaseQueue],
     threads: ReleaseThreads,
-    log: type[Log],
+    log: type[Audit],
 ) -> type[CancelJob] | None:
     """Schedule task function that generate thread of workflow task release
     method in background. This function do the same logic as the workflow poke
     method, but it runs with map of schedules and the on values.
 
-        This schedule task start runs every minute at ':02' second and it does
+        This schedule task start runs every minute at ':02' second, and it does
     not allow you to run with offset time.
 
     :param tasks: A list of WorkflowTask object.
@@ -414,15 +477,16 @@ def schedule_task(
     #   function. It will deplicate running with different schedule value
     #   because I use current time in this condition.
     #
-    #       For example, if a workflow A queue has '00:02:00' time that
-    #   should to run and its schedule has '*/2 * * * *' and '*/35 * * * *'.
-    #   This condition will release with 2 threading job.
+    #       For example, if a queue has a time release be '00:02:00' that should
+    #   to run and its schedule has '*/2 * * * *' and '*/35 * * * *'.
+    #   This condition make this function create 2 threading tasks.
     #
-    #   '00:02:00'  --> '*/2 * * * *'   --> running
-    #               --> '*/35 * * * *'  --> skip
+    #       '00:02:00'  --> '*/2 * * * *'   --> run
+    #                   --> '*/35 * * * *'  --> skip
     #
     for task in tasks:
 
+        # NOTE: Get the ReleaseQueue with an alias of the WorkflowTask.
         q: ReleaseQueue = queue[task.alias]
 
         # NOTE: Start adding queue and move the runner date in the WorkflowTask.
@@ -510,7 +574,7 @@ def schedule_control(
     stop: datetime | None = None,
     externals: DictData | None = None,
     *,
-    log: type[Log] | None = None,
+    log: type[Audit] | None = None,
 ) -> list[str]:  # pragma: no cov
     """Scheduler control function that run the chuck of schedules every minute
     and this function release monitoring thread for tracking undead thread in
@@ -533,7 +597,7 @@ def schedule_control(
         ) from None
 
     # NOTE: Get default logging.
-    log: type[Log] = log or get_log()
+    log: type[Audit] = log or get_audit()
     scheduler: Scheduler = Scheduler()
 
     # NOTE: Create the start and stop datetime.
@@ -548,15 +612,24 @@ def schedule_control(
         second=0, microsecond=0
     ) + timedelta(minutes=1)
 
+    tasks: list[WorkflowTask] = []
+    for name in schedules:
+        schedule: Schedule = Schedule.from_loader(name, externals=externals)
+        tasks.extend(
+            schedule.tasks(
+                start_date_waiting,
+                queue=queue,
+                externals=externals,
+            ),
+        )
+
     # NOTE: This schedule job will start every minute at :02 seconds.
     (
         scheduler.every(1)
         .minutes.at(":02")
         .do(
             schedule_task,
-            tasks=Schedule.extract_tasks(
-                schedules, start_date_waiting, queue, externals=externals
-            ),
+            tasks=tasks,
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -596,7 +669,7 @@ def schedule_control(
                     "[SCHEDULE]: Waiting schedule release thread that still "
                     "running in background."
                 )
-                delay(15)
+                delay(10)
                 monitor(threads)
 
             break

ddeutil/workflow/stage.py

@@ -45,7 +45,7 @@ from .__types import DictData, DictStr, TupleStr
 from .conf import config, get_logger
 from .exceptions import StageException
 from .hook import TagFunc, extract_hook
-from .result import Result
+from .result import Result, Status
 from .templates import not_in_template, param2template
 from .utils import (
     cut_id,
@@ -121,19 +121,26 @@ class BaseStage(BaseModel, ABC):
         return self
 
     @abstractmethod
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
     def handler_execute(
-        self, params: DictData, *, run_id: str | None = None
+        self,
+        params: DictData,
+        *,
+        run_id: str | None = None,
+        result: Result | None = None,
     ) -> Result:
         """Handler result from the stage execution.
 
@@ -158,23 +165,25 @@ class BaseStage(BaseModel, ABC):
         from current stage ID before release the final result.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param run_id: (str) A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        if not run_id:
-            run_id: str = gen_id(self.name + (self.id or ""), unique=True)
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=(
+                    run_id or gen_id(self.name + (self.id or ""), unique=True)
+                ),
+            )
 
-        rs_raise: Result = Result(status=1, run_id=run_id)
         try:
             # NOTE: Start calling origin function with a passing args.
-            return self.execute(params, run_id=run_id)
+            return self.execute(params, result=result)
         except Exception as err:
             # NOTE: Start catching error from the stage execution.
-            logger.error(
-                f"({cut_id(run_id)}) [STAGE]: {err.__class__.__name__}: "
-                f"{err}"
-            )
+            result.trace.error(f"[STAGE]: {err.__class__.__name__}: {err}")
             if config.stage_raise_error:
                 # NOTE: If error that raise from stage execution course by
                 #   itself, it will return that error with previous
@@ -190,8 +199,8 @@ class BaseStage(BaseModel, ABC):
 
             # NOTE: Catching exception error object to result with
             #   error_message and error keys.
-            return rs_raise.catch(
-                status=1,
+            return result.catch(
+                status=Status.FAILED,
                 context={
                     "error": err,
                     "error_message": f"{err.__class__.__name__}: {err}",
@@ -295,7 +304,9 @@ class EmptyStage(BaseStage):
         ge=0,
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
         it does not get any error from logging function.
@@ -305,22 +316,21 @@ class EmptyStage(BaseStage):
 
         :param params: A context data that want to add output result. But this
             stage does not pass any output.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Empty-Execute: {self.name!r}: "
+        result.trace.info(
+            f"[STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
         if self.sleep > 0:
             if self.sleep > 30:
-                logger.info(
-                    f"({cut_id(run_id)}) [STAGE]: ... sleep "
-                    f"({self.sleep} seconds)"
-                )
+                result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
             time.sleep(self.sleep)
-        return Result(status=0, context={}, run_id=run_id)
+
+        return result.catch(status=Status.SUCCESS)
 
 
 class BashStage(BaseStage):
@@ -334,7 +344,7 @@ class BashStage(BaseStage):
 
     Data Validate:
         >>> stage = {
-        ...     "name": "Shell stage execution",
+        ...     "name": "The Shell stage execution",
         ...     "bash": 'echo "Hello $FOO"',
         ...     "env": {
         ...         "FOO": "BAR",
@@ -391,20 +401,25 @@ class BashStage(BaseStage):
         # Note: Remove .sh file that use to run bash.
         Path(f"./{f_name}").unlink()
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
         bash: str = param2template(dedent(self.bash), params)
 
-        logger.info(f"({cut_id(run_id)}) [STAGE]: Shell-Execute: {self.name}")
+        result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
         with self.create_sh_file(
-            bash=bash, env=param2template(self.env, params), run_id=run_id
+            bash=bash,
+            env=param2template(self.env, params),
+            run_id=result.run_id,
         ) as sh:
             rs: CompletedProcess = subprocess.run(
                 sh, shell=False, capture_output=True, text=True
@@ -420,14 +435,13 @@ class BashStage(BaseStage):
                 f"Subprocess: {err}\nRunning Statement:\n---\n"
                 f"```bash\n{bash}\n```"
             )
-        return Result(
-            status=0,
+        return result.catch(
+            status=Status.SUCCESS,
             context={
                 "return_code": rs.returncode,
                 "stdout": rs.stdout.rstrip("\n") or None,
                 "stderr": rs.stderr.rstrip("\n") or None,
             },
-            run_id=run_id,
         )
 
 
@@ -492,12 +506,15 @@ class PyStage(BaseStage):
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
 
         :param params: A parameter that want to pass before run any statement.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
@@ -511,16 +528,14 @@ class PyStage(BaseStage):
         lc: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logger.info(f"({cut_id(run_id)}) [STAGE]: Py-Execute: {self.name}")
+        result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
 
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
         exec(run, _globals, lc)
 
-        return Result(
-            status=0,
-            context={"locals": lc, "globals": _globals},
-            run_id=run_id,
+        return result.catch(
+            status=Status.SUCCESS, context={"locals": lc, "globals": _globals}
         )
 
 
@@ -552,7 +567,9 @@ class HookStage(BaseStage):
         alias="with",
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Execute the Hook function that already in the hook registry.
 
         :raise ValueError: When the necessary arguments of hook function do not
@@ -562,7 +579,8 @@ class HookStage(BaseStage):
 
         :param params: A parameter that want to pass before run any statement.
         :type params: DictData
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
         :type: str | None
 
         :rtype: Result
@@ -571,7 +589,7 @@ class HookStage(BaseStage):
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = param2template(self.args, params)
+        args: DictData = {"result": result} | param2template(self.args, params)
         ips = inspect.signature(t_func)
         if any(
             (k.removeprefix("_") not in args and k not in args)
@@ -587,10 +605,10 @@ class HookStage(BaseStage):
             if k.removeprefix("_") in args:
                 args[k] = args.pop(k.removeprefix("_"))
 
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Hook-Execute: "
-            f"{t_func.name}@{t_func.tag}"
-        )
+        if "result" not in ips.parameters:
+            args.pop("result")
+
+        result.trace.info(f"[STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}")
         rs: DictData = t_func(**param2template(args, params))
 
         # VALIDATE:
@@ -600,7 +618,7 @@ class HookStage(BaseStage):
                 f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
                 f"to result model, you change return type to `dict`."
             )
-        return Result(status=0, context=rs, run_id=run_id)
+        return result.catch(status=Status.SUCCESS, context=rs)
 
 
 class TriggerStage(BaseStage):
@@ -626,12 +644,15 @@ class TriggerStage(BaseStage):
         description="A parameter that want to pass to workflow execution.",
     )
 
-    def execute(self, params: DictData, *, run_id: str | None = None) -> Result:
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
 
         :param params: A parameter data that want to use in this execution.
-        :param run_id: A running stage ID for this execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
 
         :rtype: Result
         """
@@ -644,13 +665,11 @@ class TriggerStage(BaseStage):
         # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
         wf: Workflow = Workflow.from_loader(name=_trigger)
-        logger.info(
-            f"({cut_id(run_id)}) [STAGE]: Trigger-Execute: {_trigger!r}"
-        )
+        result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
         return wf.execute(
             params=param2template(self.params, params),
-            run_id=run_id,
-        ).set_run_id(run_id)
+            result=result,
+        )
 
 
 # NOTE:
@@ -668,12 +687,12 @@ Stage = Union[
 
 
 # TODO: Not implement this stages yet
-class ParallelStage(BaseModel):
+class ParallelStage(BaseModel):  # pragma: no cov
     parallel: list[Stage]
     max_parallel_core: int = Field(default=2)
 
 
 # TODO: Not implement this stages yet
-class ForEachStage(BaseModel):
+class ForEachStage(BaseModel):  # pragma: no cov
     foreach: list[str]
     stages: list[Stage]

ddeutil/workflow/templates.py

@@ -79,7 +79,7 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
 def make_filter_registry() -> dict[str, FilterRegistry]:
     """Return registries of all functions that able to called with task.
 
-    :rtype: dict[str, Registry]
+    :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
     for module in config.regis_filter:
@@ -108,6 +108,8 @@ def get_args_const(
 ) -> tuple[str, list[Constant], dict[str, Constant]]:
     """Get arguments and keyword-arguments from function calling string.
 
+    :param expr: An expr string value.
+
     :rtype: tuple[str, list[Constant], dict[str, Constant]]
     """
     try:
@@ -150,6 +152,11 @@ def get_args_from_filter(
 ) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]:  # pragma: no cov
     """Get arguments and keyword-arguments from filter function calling string.
     and validate it with the filter functions mapping dict.
+
+    :param ft:
+    :param filters:
+
+    :rtype: tuple[str, FilterRegistry, list[Any], dict[Any, Any]]
     """
     func_name, _args, _kwargs = get_args_const(ft)
     args: list[Any] = [arg.value for arg in _args]
@@ -243,7 +250,7 @@ def str2template(
     params: DictData,
     *,
     filters: dict[str, FilterRegistry] | None = None,
-) -> Any:
+) -> str:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
 
@@ -255,6 +262,8 @@ def str2template(
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters:
+
+    :rtype: str
     """
     filters: dict[str, FilterRegistry] = filters or make_filter_registry()
 
@@ -295,7 +304,7 @@ def str2template(
     return search_env_replace(value)
 
 
-def param2template(value: Any, params: DictData) -> Any:
+def param2template(value: T, params: DictData) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
@@ -303,7 +312,7 @@ def param2template(value: Any, params: DictData) -> Any:
     :param params: A parameter value that getting with matched regular
         expression.
 
-    :rtype: Any
+    :rtype: T
     :returns: An any getter value from the params input.
     """
     filters: dict[str, FilterRegistry] = make_filter_registry()