ddeutil-workflow 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl

ddeutil/workflow/job.py

@@ -30,13 +30,12 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import config
+from .conf import config, get_logger
 from .exceptions import (
     JobException,
     StageException,
     UtilException,
 )
-from .log import get_logger
 from .stage import Stage
 from .utils import (
     Result,
@@ -111,6 +110,7 @@ def make(
             all(inc.get(k) == v for k, v in m.items()) for m in [*final, *add]
         ):
             continue
+
         add.append(inc)
 
     # NOTE: Merge all matrix together.
@@ -273,11 +273,32 @@ class Job(BaseModel):
 
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
-        """Prepare description string that was created on a template."""
+        """Prepare description string that was created on a template.
+
+        :rtype: str
+        """
         return dedent(value)
 
+    @field_validator("stages", mode="after")
+    def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
+        """Validate a stage ID of all stage in stages field should not be
+        duplicate.
+
+        :rtype: list[Stage]
+        """
+        # VALIDATE: Validate stage id should not duplicate.
+        rs: list[str] = []
+        for stage in value:
+            name: str = stage.id or stage.name
+            if name in rs:
+                raise ValueError(
+                    "Stage name in jobs object should not be duplicate."
+                )
+            rs.append(name)
+        return value
+
     @model_validator(mode="after")
-    def __prepare_running_id__(self) -> Self:
+    def __prepare_running_id_and_stage_name__(self) -> Self:
         """Prepare the job running ID.
 
         :rtype: Self
@@ -355,7 +376,7 @@ class Job(BaseModel):
         to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
-            else output[next(iter(output))]
+            else output.get(next(iter(output), "DUMMY"), {})
         )
         return to
 
@@ -365,7 +386,6 @@ class Job(BaseModel):
         params: DictData,
         *,
         event: Event | None = None,
-        raise_error: bool = True,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
         workflow execution to strategy matrix.
@@ -374,19 +394,20 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
-        :raise JobException: If it has any error from StageException or
-            UtilException.
+        :raise JobException: If it has any error from ``StageException`` or
+            ``UtilException``.
 
         :param strategy: A metrix strategy value.
         :param params: A dynamic parameters.
         :param event: An manger event that pass to the PoolThreadExecutor.
-        :param raise_error: A flag that raise error instead catching to result
-            if it get exception from stage execution.
+
         :rtype: Result
         """
         strategy_id: str = gen_id(strategy)
 
-        # NOTE: Create strategy execution context and update a matrix and copied
+        # PARAGRAPH:
+        #
+        #       Create strategy execution context and update a matrix and copied
         #   of params. So, the context value will have structure like;
         #
         #   {
@@ -405,14 +426,14 @@ class Job(BaseModel):
             # IMPORTANT: Change any stage running IDs to this job running ID.
             stage: Stage = stage.get_running_id(self.run_id)
 
-            _st_name: str = stage.id or stage.name
+            name: str = stage.id or stage.name
 
             if stage.is_skipped(params=context):
-                logger.info(f"({self.run_id}) [JOB]: Skip stage: {_st_name!r}")
+                logger.info(f"({self.run_id}) [JOB]: Skip stage: {name!r}")
                 continue
 
             logger.info(
-                f"({self.run_id}) [JOB]: Start execute the stage: {_st_name!r}"
+                f"({self.run_id}) [JOB]: Start execute the stage: {name!r}"
             )
 
             # NOTE: Logging a matrix that pass on this stage execution.
@@ -432,20 +453,20 @@ class Job(BaseModel):
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
                             "stages": context.pop("stages", {}),
-                            # NOTE: Set the error keys.
                             "error": JobException(
-                                "Process Event stopped before execution"
+                                "Job strategy was canceled from trigger event "
+                                "that had stopped before execution."
+                            ),
+                            "error_message": (
+                                "Job strategy was canceled from trigger event "
+                                "that had stopped before execution."
                             ),
-                            "error_message": {
-                                "message": (
-                                    "Process Event stopped before execution"
-                                ),
-                            },
                         },
                     },
                 )
 
-            # NOTE:
+            # PARAGRAPH:
+            #
             #       I do not use below syntax because `params` dict be the
             #   reference memory pointer and it was changed when I action
             #   anything like update or re-construct this.
@@ -471,16 +492,25 @@ class Job(BaseModel):
                 logger.error(
                     f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
-                if raise_error:
+                if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                else:
-                    raise NotImplementedError() from None
+                return Result(
+                    status=1,
+                    context={
+                        strategy_id: {
+                            "matrix": strategy,
+                            "stages": context.pop("stages", {}),
+                            "error": err,
+                            "error_message": f"{err.__class__.__name__}: {err}",
+                        },
+                    },
+                )
 
-            # NOTE: Remove new stage object that was created from
-            #   ``get_running_id`` method.
+            # NOTE: Remove the current stage object that was created from
+            #   ``get_running_id`` method for saving memory.
             del stage
 
         return Result(
@@ -583,30 +613,34 @@ class Job(BaseModel):
         )
         logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
-        # NOTE: Stop all running tasks with setting the event manager and cancel
+        # NOTE:
+        #       Stop all running tasks with setting the event manager and cancel
         #   any scheduled tasks.
+        #
         if len(done) != len(futures):
             event.set()
-            for future in futures:
+            for future in not_done:
                 future.cancel()
 
-            del future
-
+        future: Future
         for future in done:
-            if future.exception():
-                status = 1
+            if err := future.exception():
+                status: int = 1
                 logger.error(
                     f"({self.run_id}) [JOB]: One stage failed with: "
                     f"{future.exception()}, shutting down this future."
                 )
-            elif future.cancelled():
+                context.update(
+                    {
+                        "error": err,
+                        "error_message": f"{err.__class__.__name__}: {err}",
+                    },
+                )
                 continue
 
             # NOTE: Update the result context to main job context.
             context.update(future.result(timeout=result_timeout).context)
 
-            del future
-
         return rs_final.catch(status=status, context=context)
 
     def __catch_all_completed(
@@ -631,7 +665,7 @@ class Job(BaseModel):
         for future in as_completed(futures, timeout=timeout):
             try:
                 context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:
+            except TimeoutError:  # pragma: no cov
                 status = 1
                 logger.warning(
                     f"({self.run_id}) [JOB]: Task is hanging. Attempting to "
@@ -653,6 +687,10 @@ class Job(BaseModel):
                     f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
                     f"{err}"
                 )
-            finally:
-                del future
+                context.update(
+                    {
+                        "error": err,
+                        "error_message": f"{err.__class__.__name__}: {err}",
+                    },
+                )
         return rs_final.catch(status=status, context=context)

ddeutil/workflow/on.py

@@ -14,9 +14,9 @@ from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
+from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
 from .__types import DictData, DictStr, TupleStr
 from .conf import Loader
-from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
 
 __all__: TupleStr = (
     "On",
@@ -109,7 +109,7 @@ class On(BaseModel):
     def from_loader(
         cls,
         name: str,
-        externals: DictData,
+        externals: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config that will use loader object for
         getting the data.
@@ -117,6 +117,7 @@ class On(BaseModel):
         :param name: A name of config that will getting from loader.
         :param externals: A extras external parameter that will keep in extras.
         """
+        externals: DictData = externals or {}
         loader: Loader = Loader(name, externals=externals)
 
         # NOTE: Validate the config type match with current connection model
@@ -139,7 +140,9 @@ class On(BaseModel):
                 )
             )
         if "cronjob" not in loader_data:
-            raise ValueError("Config does not set ``cronjob`` key")
+            raise ValueError(
+                "Config does not set ``cronjob`` or ``interval`` keys"
+            )
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
@@ -175,17 +178,17 @@ class On(BaseModel):
 
     def generate(self, start: str | datetime) -> CronRunner:
         """Return Cron runner object."""
-        if not isinstance(start, datetime):
+        if isinstance(start, str):
             start: datetime = datetime.fromisoformat(start)
+        elif not isinstance(start, datetime):
+            raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
     def next(self, start: str | datetime) -> datetime:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
         """
-        if not isinstance(start, datetime):
-            start: datetime = datetime.fromisoformat(start)
-        return self.cronjob.schedule(date=start, tz=self.tz).next
+        return self.generate(start=start).next
 
 
 class YearOn(On):

ddeutil/workflow/repeat.py

@@ -12,9 +12,8 @@ from functools import wraps
 
 from starlette.concurrency import run_in_threadpool
 
-from .conf import config
-from .cron import CronJob
-from .log import get_logger
+from .__cron import CronJob
+from .conf import config, get_logger
 
 logger = get_logger("ddeutil.workflow")
 

ddeutil/workflow/route.py

@@ -16,8 +16,7 @@ from pydantic import BaseModel
 
 from . import Workflow
 from .__types import DictData
-from .conf import Loader, config
-from .log import get_logger
+from .conf import Loader, config, get_logger
 from .scheduler import Schedule
 from .utils import Result
 

ddeutil/workflow/scheduler.py

@@ -52,15 +52,14 @@ except ImportError:
 
 try:
     from schedule import CancelJob
-except ImportError:
+except ImportError:  # pragma: no cov
     CancelJob = None
 
+from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, config
-from .cron import CronRunner
+from .conf import FileLog, Loader, Log, config, get_logger
 from .exceptions import JobException, WorkflowException
 from .job import Job
-from .log import FileLog, Log, get_logger
 from .on import On
 from .utils import (
     Param,
@@ -230,8 +229,8 @@ class Workflow(BaseModel):
                 need for need in self.jobs[job].needs if need not in self.jobs
             ]:
                 raise WorkflowException(
-                    f"This needed jobs: {not_exist} do not exist in this "
-                    f"workflow, {self.name!r}"
+                    f"The needed jobs: {not_exist} do not found in "
+                    f"{self.name!r}."
                 )
 
             # NOTE: update a job id with its job id from workflow template
@@ -354,11 +353,11 @@ class Workflow(BaseModel):
         # NOTE: get next schedule time that generate from now.
         next_time: datetime = gen.next
 
-        # NOTE: get next utils it does not logger.
+        # NOTE: While-loop to getting next until it does not logger.
         while log.is_pointed(self.name, next_time, queue=queue):
             next_time: datetime = gen.next
 
-        # NOTE: push this next running time to log queue
+        # NOTE: Heap-push this next running time to log queue list.
         heappush(queue, next_time)
 
         # VALIDATE: Check the different time between the next schedule time and
@@ -377,7 +376,7 @@ class Workflow(BaseModel):
                 status=0,
                 context={
                     "params": params,
-                    "poking": {"skipped": [str(on.cronjob)], "run": []},
+                    "release": {"status": "skipped", "cron": [str(on.cronjob)]},
                 },
             )
 
@@ -389,7 +388,7 @@ class Workflow(BaseModel):
         # NOTE: Release when the time is nearly to schedule time.
         while (duration := get_diff_sec(next_time, tz=cron_tz)) > (
             sleep_interval + 5
-        ):
+        ):  # pragma: no cov
             logger.debug(
                 f"({self.run_id}) [CORE]: {self.name!r} : {on.cronjob} : "
                 f"Sleep until: {duration}"
@@ -440,7 +439,7 @@ class Workflow(BaseModel):
             status=0,
             context={
                 "params": params,
-                "poking": {"skipped": [], "run": [str(on.cronjob)]},
+                "release": {"status": "run", "cron": [str(on.cronjob)]},
             },
         )
 
@@ -493,7 +492,7 @@ class Workflow(BaseModel):
             for future in as_completed(futures):
                 results.append(future.result(timeout=60))
 
-        if len(queue) > 0:
+        if len(queue) > 0:  # pragma: no cov
             logger.error(
                 f"({self.run_id}) [POKING]: Log Queue does empty when poking "
                 f"process was finishing."
@@ -709,7 +708,7 @@ class Workflow(BaseModel):
                     raise WorkflowException(f"{err}")
                 try:
                     future.result(timeout=60)
-                except TimeoutError as err:
+                except TimeoutError as err:  # pragma: no cove
                     raise WorkflowException(
                         "Timeout when getting result from future"
                     ) from err
@@ -718,11 +717,11 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
+        logger.warning(  # pragma: no cov
             f"({self.run_id}) [WORKFLOW]: Execution of workflow, {self.name!r} "
             f", was timeout"
         )
-        raise WorkflowException(
+        raise WorkflowException(  # pragma: no cov
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -766,7 +765,8 @@ class Workflow(BaseModel):
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
-            #   before release.
+            #   before release. This job execution process will running until
+            #   done before checking all execution timeout or not.
             #
             #   {
             #       'params': <input-params>,
@@ -784,10 +784,10 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
+        logger.warning(  # pragma: no cov
             f"({self.run_id}) [WORKFLOW]: Execution of workflow was timeout"
         )
-        raise WorkflowException(
+        raise WorkflowException(  # pragma: no cov
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -833,12 +833,13 @@ class ScheduleWorkflow(BaseModel):
         if on := data.pop("on", []):
 
             if isinstance(on, str):
-                on = [on]
+                on: list[str] = [on]
 
             if any(not isinstance(n, (dict, str)) for n in on):
                 raise TypeError("The ``on`` key should be list of str or dict")
 
-            # NOTE: Pass on value to Loader and keep on model object to on field
+            # NOTE: Pass on value to Loader and keep on model object to on
+            #   field.
             data["on"] = [
                 (
                     Loader(n, externals=(externals or {})).data
@@ -903,12 +904,14 @@ class Schedule(BaseModel):
         *,
         externals: DictData | None = None,
     ) -> list[WorkflowTaskData]:
-        """Generate Task from the current datetime.
+        """Return the list of WorkflowTaskData object from the specific input
+        datetime that mapping with the on field.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
         :param running: A mapping of name and list of datetime for running.
         :param externals: An external parameters that pass to the Loader object.
+
         :rtype: list[WorkflowTaskData]
         """
 
@@ -923,12 +926,14 @@ class Schedule(BaseModel):
             queue[wfs.name]: list[datetime] = []
             running[wfs.name]: list[datetime] = []
 
-            # NOTE: Create default on if it does not passing on the Schedule.
+            # NOTE: Create the default on value if it does not passing on the
+            #   Schedule object.
             _ons: list[On] = wf.on.copy() if len(wfs.on) == 0 else wfs.on
 
             for on in _ons:
-                on_gen = on.generate(start_date)
+                on_gen: CronRunner = on.generate(start_date)
                 next_running_date = on_gen.next
+
                 while next_running_date in queue[wfs.name]:
                     next_running_date = on_gen.next
 
@@ -958,13 +963,14 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
     :param cancel_on_failure: A flag that allow to return the CancelJob or not
         it will raise.
-    :rtype: Callable[P, Optional[CancelJob]]
+
+    :rtype: DecoratorCancelJob
     """
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:
         try:
             # NOTE: Check the function that want to handle is method or not.
-            if inspect.ismethod(func):
+            if inspect.ismethod(func):  # pragma: no cov
 
                 @wraps(func)
                 def wrapper(self, *args, **kwargs):
@@ -978,7 +984,7 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
             return wrapper
 
-        except Exception as err:
+        except Exception as err:  # pragma: no cov
             logger.exception(err)
             if cancel_on_failure:
                 return CancelJob
@@ -1006,7 +1012,7 @@ class WorkflowTaskData:
         *,
         waiting_sec: int = 60,
         sleep_interval: int = 15,
-    ) -> None:
+    ) -> None:  # pragma: no cov
         """Workflow release, it will use with the same logic of
         `workflow.release` method.
 
@@ -1120,7 +1126,7 @@ class WorkflowTaskData:
             future_running_time in self.running[wf.name]
             or future_running_time in self.queue[wf.name]
             or future_running_time < finish_time
-        ):
+        ):  # pragma: no cov
             future_running_time: datetime = gen.next
 
         heappush(self.queue[wf.name], future_running_time)
@@ -1135,7 +1141,7 @@ class WorkflowTaskData:
         return NotImplemented
 
 
-@catch_exceptions(cancel_on_failure=True)
+@catch_exceptions(cancel_on_failure=True)  # pragma: no cov
 def workflow_task(
     workflow_tasks: list[WorkflowTaskData],
     stop: datetime,
@@ -1234,7 +1240,7 @@ def workflow_task(
     logger.debug(f"[WORKFLOW]: {'=' * 100}")
 
 
-def workflow_monitor(threads: dict[str, Thread]) -> None:
+def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
     """Workflow schedule for monitoring long running thread from the schedule
     control.
 
@@ -1256,7 +1262,7 @@ def workflow_control(
     schedules: list[str],
     stop: datetime | None = None,
     externals: DictData | None = None,
-) -> list[str]:
+) -> list[str]:  # pragma: no cov
     """Workflow scheduler control.
 
     :param schedules: A list of workflow names that want to schedule running.
@@ -1344,7 +1350,7 @@ def workflow_runner(
     stop: datetime | None = None,
     externals: DictData | None = None,
     excluded: list[str] | None = None,
-) -> list[str]:
+) -> list[str]:  # pragma: no cov
     """Workflow application that running multiprocessing schedule with chunk of
     workflows that exists in config path.