ddeutil-workflow 0.0.16__py3-none-any.whl → 0.0.17__py3-none-any.whl

ddeutil/workflow/job.py

@@ -30,13 +30,12 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import config
+from .conf import config, get_logger
 from .exceptions import (
     JobException,
     StageException,
     UtilException,
 )
-from .log import get_logger
 from .stage import Stage
 from .utils import (
     Result,
@@ -111,6 +110,7 @@ def make(
             all(inc.get(k) == v for k, v in m.items()) for m in [*final, *add]
         ):
             continue
+
         add.append(inc)
 
     # NOTE: Merge all matrix together.
@@ -273,11 +273,32 @@ class Job(BaseModel):
 
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
-        """Prepare description string that was created on a template."""
+        """Prepare description string that was created on a template.
+
+        :rtype: str
+        """
         return dedent(value)
 
+    @field_validator("stages", mode="after")
+    def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
+        """Validate a stage ID of all stage in stages field should not be
+        duplicate.
+
+        :rtype: list[Stage]
+        """
+        # VALIDATE: Validate stage id should not duplicate.
+        rs: list[str] = []
+        for stage in value:
+            name: str = stage.id or stage.name
+            if name in rs:
+                raise ValueError(
+                    "Stage name in jobs object should not be duplicate."
+                )
+            rs.append(name)
+        return value
+
     @model_validator(mode="after")
-    def __prepare_running_id__(self) -> Self:
+    def __prepare_running_id_and_stage_name__(self) -> Self:
         """Prepare the job running ID.
 
         :rtype: Self
@@ -355,7 +376,7 @@ class Job(BaseModel):
         to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
-            else output[next(iter(output))]
+            else output.get(next(iter(output), "DUMMY"), {})
         )
         return to
 
@@ -365,7 +386,6 @@ class Job(BaseModel):
         params: DictData,
         *,
         event: Event | None = None,
-        raise_error: bool = True,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
         workflow execution to strategy matrix.
@@ -374,19 +394,20 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
-        :raise JobException: If it has any error from StageException or
-            UtilException.
+        :raise JobException: If it has any error from ``StageException`` or
+            ``UtilException``.
 
         :param strategy: A metrix strategy value.
         :param params: A dynamic parameters.
         :param event: An manger event that pass to the PoolThreadExecutor.
-        :param raise_error: A flag that raise error instead catching to result
-            if it get exception from stage execution.
+
         :rtype: Result
         """
         strategy_id: str = gen_id(strategy)
 
-        # NOTE: Create strategy execution context and update a matrix and copied
+        # PARAGRAPH:
+        #
+        #       Create strategy execution context and update a matrix and copied
         #   of params. So, the context value will have structure like;
         #
         #   {
@@ -405,14 +426,14 @@ class Job(BaseModel):
             # IMPORTANT: Change any stage running IDs to this job running ID.
             stage: Stage = stage.get_running_id(self.run_id)
 
-            _st_name: str = stage.id or stage.name
+            name: str = stage.id or stage.name
 
             if stage.is_skipped(params=context):
-                logger.info(f"({self.run_id}) [JOB]: Skip stage: {_st_name!r}")
+                logger.info(f"({self.run_id}) [JOB]: Skip stage: {name!r}")
                 continue
 
             logger.info(
-                f"({self.run_id}) [JOB]: Start execute the stage: {_st_name!r}"
+                f"({self.run_id}) [JOB]: Start execute the stage: {name!r}"
             )
 
             # NOTE: Logging a matrix that pass on this stage execution.
@@ -432,20 +453,20 @@ class Job(BaseModel):
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
                             "stages": context.pop("stages", {}),
-                            # NOTE: Set the error keys.
                             "error": JobException(
-                                "Process Event stopped before execution"
+                                "Job strategy was canceled from trigger event "
+                                "that had stopped before execution."
+                            ),
+                            "error_message": (
+                                "Job strategy was canceled from trigger event "
+                                "that had stopped before execution."
                             ),
-                            "error_message": {
-                                "message": (
-                                    "Process Event stopped before execution"
-                                ),
-                            },
                         },
                     },
                 )
 
-            # NOTE:
+            # PARAGRAPH:
+            #
             #       I do not use below syntax because `params` dict be the
             #   reference memory pointer and it was changed when I action
             #   anything like update or re-construct this.
@@ -471,16 +492,25 @@ class Job(BaseModel):
                 logger.error(
                     f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
-                if raise_error:
+                if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                else:
-                    raise NotImplementedError() from None
+                return Result(
+                    status=1,
+                    context={
+                        strategy_id: {
+                            "matrix": strategy,
+                            "stages": context.pop("stages", {}),
+                            "error": err,
+                            "error_message": f"{err.__class__.__name__}: {err}",
+                        },
+                    },
+                )
 
-            # NOTE: Remove new stage object that was created from
-            #   ``get_running_id`` method.
+            # NOTE: Remove the current stage object that was created from
+            #   ``get_running_id`` method for saving memory.
             del stage
 
         return Result(
@@ -523,6 +553,8 @@ class Job(BaseModel):
         # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
 
+        print("Job Run Fail-Fast:", self.strategy.fail_fast)
+
         # IMPORTANT: Start running strategy execution by multithreading because
         #   it will running by strategy values without waiting previous
         #   execution.
@@ -583,30 +615,34 @@ class Job(BaseModel):
         )
         logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
-        # NOTE: Stop all running tasks with setting the event manager and cancel
+        # NOTE:
+        #       Stop all running tasks with setting the event manager and cancel
         #   any scheduled tasks.
+        #
         if len(done) != len(futures):
             event.set()
-            for future in futures:
+            for future in not_done:
                 future.cancel()
 
-            del future
-
+        future: Future
         for future in done:
-            if future.exception():
-                status = 1
+            if err := future.exception():
+                status: int = 1
                 logger.error(
                     f"({self.run_id}) [JOB]: One stage failed with: "
                     f"{future.exception()}, shutting down this future."
                 )
-            elif future.cancelled():
+                context.update(
+                    {
+                        "error": err,
+                        "error_message": f"{err.__class__.__name__}: {err}",
+                    },
+                )
                 continue
 
             # NOTE: Update the result context to main job context.
             context.update(future.result(timeout=result_timeout).context)
 
-            del future
-
         return rs_final.catch(status=status, context=context)
 
     def __catch_all_completed(
@@ -631,7 +667,7 @@ class Job(BaseModel):
         for future in as_completed(futures, timeout=timeout):
             try:
                 context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:
+            except TimeoutError:  # pragma: no cov
                 status = 1
                 logger.warning(
                     f"({self.run_id}) [JOB]: Task is hanging. Attempting to "
@@ -653,6 +689,10 @@ class Job(BaseModel):
                     f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
                     f"{err}"
                 )
-            finally:
-                del future
+                context.update(
+                    {
+                        "error": err,
+                        "error_message": f"{err.__class__.__name__}: {err}",
+                    },
+                )
         return rs_final.catch(status=status, context=context)

ddeutil/workflow/on.py

@@ -14,9 +14,9 @@ from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
+from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
 from .__types import DictData, DictStr, TupleStr
 from .conf import Loader
-from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
 
 __all__: TupleStr = (
     "On",
@@ -109,7 +109,7 @@ class On(BaseModel):
     def from_loader(
         cls,
         name: str,
-        externals: DictData,
+        externals: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config that will use loader object for
         getting the data.
@@ -117,6 +117,7 @@ class On(BaseModel):
         :param name: A name of config that will getting from loader.
         :param externals: A extras external parameter that will keep in extras.
         """
+        externals: DictData = externals or {}
         loader: Loader = Loader(name, externals=externals)
 
         # NOTE: Validate the config type match with current connection model
@@ -139,7 +140,9 @@ class On(BaseModel):
                 )
             )
         if "cronjob" not in loader_data:
-            raise ValueError("Config does not set ``cronjob`` key")
+            raise ValueError(
+                "Config does not set ``cronjob`` or ``interval`` keys"
+            )
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
@@ -175,17 +178,17 @@ class On(BaseModel):
 
     def generate(self, start: str | datetime) -> CronRunner:
         """Return Cron runner object."""
-        if not isinstance(start, datetime):
+        if isinstance(start, str):
             start: datetime = datetime.fromisoformat(start)
+        elif not isinstance(start, datetime):
+            raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
     def next(self, start: str | datetime) -> datetime:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
         """
-        if not isinstance(start, datetime):
-            start: datetime = datetime.fromisoformat(start)
-        return self.cronjob.schedule(date=start, tz=self.tz).next
+        return self.generate(start=start).next
 
 
 class YearOn(On):

ddeutil/workflow/repeat.py

@@ -12,9 +12,8 @@ from functools import wraps
 
 from starlette.concurrency import run_in_threadpool
 
-from .conf import config
+from .conf import config, get_logger
 from .cron import CronJob
-from .log import get_logger
 
 logger = get_logger("ddeutil.workflow")
 

ddeutil/workflow/route.py

@@ -16,8 +16,7 @@ from pydantic import BaseModel
 
 from . import Workflow
 from .__types import DictData
-from .conf import Loader, config
-from .log import get_logger
+from .conf import Loader, config, get_logger
 from .scheduler import Schedule
 from .utils import Result
 

ddeutil/workflow/scheduler.py

@@ -52,15 +52,14 @@ except ImportError:
 
 try:
     from schedule import CancelJob
-except ImportError:
+except ImportError:  # pragma: no cov
     CancelJob = None
 
+from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, config
-from .cron import CronRunner
+from .conf import FileLog, Loader, Log, config, get_logger
 from .exceptions import JobException, WorkflowException
 from .job import Job
-from .log import FileLog, Log, get_logger
 from .on import On
 from .utils import (
     Param,
@@ -230,8 +229,8 @@ class Workflow(BaseModel):
                 need for need in self.jobs[job].needs if need not in self.jobs
             ]:
                 raise WorkflowException(
-                    f"This needed jobs: {not_exist} do not exist in this "
-                    f"workflow, {self.name!r}"
+                    f"The needed jobs: {not_exist} do not found in "
+                    f"{self.name!r}."
                 )
 
             # NOTE: update a job id with its job id from workflow template
@@ -354,11 +353,11 @@ class Workflow(BaseModel):
         # NOTE: get next schedule time that generate from now.
         next_time: datetime = gen.next
 
-        # NOTE: get next utils it does not logger.
+        # NOTE: While-loop to getting next until it does not logger.
         while log.is_pointed(self.name, next_time, queue=queue):
             next_time: datetime = gen.next
 
-        # NOTE: push this next running time to log queue
+        # NOTE: Heap-push this next running time to log queue list.
         heappush(queue, next_time)
 
         # VALIDATE: Check the different time between the next schedule time and
@@ -709,7 +708,7 @@ class Workflow(BaseModel):
                     raise WorkflowException(f"{err}")
                 try:
                     future.result(timeout=60)
-                except TimeoutError as err:
+                except TimeoutError as err:  # pragma: no cove
                     raise WorkflowException(
                         "Timeout when getting result from future"
                     ) from err

ddeutil/workflow/stage.py

@@ -13,8 +13,8 @@ handle stage error on this stage model. I think stage model should have a lot of
 usecase and it does not worry when I want to create a new one.
 
     Execution   --> Ok      --> Result with 0
-                --> Error   --> Raise StageException
-                            --> Result with 1 (if env var was set)
+                --> Error   --> Result with 1 (if env var was set)
+                            --> Raise StageException
 
     On the context I/O that pass to a stage object at execute process. The
 execute method receives a `params={"params": {...}}` value for mapping to
@@ -48,9 +48,8 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Re, TupleStr
-from .conf import config
+from .conf import config, get_logger
 from .exceptions import StageException
-from .log import get_logger
 from .utils import (
     Registry,
     Result,
@@ -94,12 +93,12 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                                         status: 0
                                         context:
                                             outputs: ...
-                    --> Error   --> Raise StageException
-                                --> Result (if env var was set)
+                    --> Error   --> Result (if env var was set)
                                         status: 1
                                         context:
                                             error: ...
                                             error_message: ...
+                    --> Error   --> Raise StageException
 
         On the last step, it will set the running ID on a return result object
     from current stage ID before release the final result.
@@ -128,6 +127,7 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
+                print("Stage Raise error:", config.stage_raise_error)
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
                     #   itself, it will return that error with previous
@@ -190,6 +190,9 @@ class BaseStage(BaseModel, ABC):
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
 
+        :raise ValueError: When the ID and name fields include matrix parameter
+            template with the 'matrix.' string value.
+
         :rtype: Self
         """
         if self.run_id is None:
@@ -199,7 +202,7 @@ class BaseStage(BaseModel, ABC):
         #   template. (allow only matrix)
         if not_in_template(self.id) or not_in_template(self.name):
             raise ValueError(
-                "Stage name and ID should only template with matrix."
+                "Stage name and ID should only template with 'matrix.'"
             )
 
         return self
@@ -236,10 +239,10 @@ class BaseStage(BaseModel, ABC):
         The result of the `to` variable will be;
 
             ... (iii) to: {
-                            'stages': {
-                                '<stage-id>': {'outputs': {'foo': 'bar'}}
-                            }
+                        'stages': {
+                            '<stage-id>': {'outputs': {'foo': 'bar'}}
                         }
+                    }
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
@@ -273,6 +276,11 @@ class BaseStage(BaseModel, ABC):
         """Return true if condition of this stage do not correct. This process
         use build-in eval function to execute the if-condition.
 
+        :raise StageException: When it has any error raise from the eval
+            condition statement.
+        :raise StageException: When return type of the eval condition statement
+            does not return with boolean type.
+
         :param params: A parameters that want to pass to condition template.
         :rtype: bool
         """
@@ -460,6 +468,15 @@ class PyStage(BaseStage):
         ),
     )
 
+    @staticmethod
+    def pick_keys_from_locals(values: DictData) -> Iterator[str]:
+        from inspect import ismodule
+
+        for value in values:
+            if value == "__annotations__" or ismodule(values[value]):
+                continue
+            yield value
+
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Override set an outputs method for the Python execution process that
         extract output from all the locals values.
@@ -469,15 +486,19 @@ class PyStage(BaseStage):
         :rtype: DictData
         """
         # NOTE: The output will fileter unnecessary keys from locals.
-        _locals: DictData = output["locals"]
+        lc: DictData = output.get("locals", {})
         super().set_outputs(
-            {k: _locals[k] for k in _locals if k != "__annotations__"}, to=to
+            (
+                {k: lc[k] for k in self.pick_keys_from_locals(lc)}
+                | {k: output[k] for k in output if k.startswith("error")}
+            ),
+            to=to,
         )
 
-        # NOTE:
-        #   Override value that changing from the globals that pass via exec.
-        _globals: DictData = output["globals"]
-        to.update({k: _globals[k] for k in to if k in _globals})
+        # NOTE: Override value that changing from the globals that pass via the
+        #   exec function.
+        gb: DictData = output.get("globals", {})
+        to.update({k: gb[k] for k in to if k in gb})
         return to
 
     @handler_result()
@@ -495,15 +516,15 @@ class PyStage(BaseStage):
         _globals: DictData = (
             globals() | params | param2template(self.vars, params)
         )
-        _locals: DictData = {}
+        lc: DictData = {}
 
         # NOTE: Start exec the run statement.
         logger.info(f"({self.run_id}) [STAGE]: Py-Execute: {self.name}")
-        exec(run, _globals, _locals)
+        exec(run, _globals, lc)
 
         return Result(
             status=0,
-            context={"locals": _locals, "globals": _globals},
+            context={"locals": lc, "globals": _globals},
         )
 
 
@@ -522,6 +543,11 @@ def extract_hook(hook: str) -> Callable[[], TagFunc]:
     """Extract Hook function from string value to hook partial function that
     does run it at runtime.
 
+    :raise NotImplementedError: When the searching hook's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching hook's tag result does not
+        exists in the registry with its function key.
+
     :param hook: A hook value that able to match with Task regex.
     :rtype: Callable[[], TagFunc]
     """
@@ -581,6 +607,11 @@ class HookStage(BaseStage):
     def execute(self, params: DictData) -> Result:
         """Execute the Hook function that already in the hook registry.
 
+        :raise ValueError: When the necessary arguments of hook function do not
+            set from the input params argument.
+        :raise TypeError: When the return type of hook function does not be
+            dict type.
+
         :param params: A parameter that want to pass before run any statement.
         :type params: DictData
         :rtype: Result