ddeutil-workflow 0.0.19__py3-none-any.whl → 0.0.21__py3-none-any.whl

ddeutil/workflow/job.py

@@ -11,7 +11,6 @@ job.
 from __future__ import annotations
 
 import copy
-import time
 from concurrent.futures import (
     FIRST_EXCEPTION,
     Future,
@@ -48,13 +47,13 @@ from .utils import (
 )
 
 logger = get_logger("ddeutil.workflow")
-MatrixInclude = list[dict[str, Union[str, int]]]
-MatrixExclude = list[dict[str, Union[str, int]]]
+MatrixFilter = list[dict[str, Union[str, int]]]
 
 
 __all__: TupleStr = (
     "Strategy",
     "Job",
+    "TriggerRules",
     "make",
 )
 
@@ -63,16 +62,20 @@ __all__: TupleStr = (
 @lru_cache
 def make(
     matrix: Matrix,
-    include: MatrixInclude,
-    exclude: MatrixExclude,
+    include: MatrixFilter,
+    exclude: MatrixFilter,
 ) -> list[DictStr]:
     """Make a list of product of matrix values that already filter with
     exclude matrix and add specific matrix with include.
 
+        This function use the `lru_cache` decorator function increase
+    performance for duplicate matrix value scenario.
+
     :param matrix: A matrix values that want to cross product to possible
         parallelism values.
     :param include: A list of additional matrix that want to adds-in.
     :param exclude: A list of exclude matrix that want to filter-out.
+
     :rtype: list[DictStr]
     """
     # NOTE: If it does not set matrix, it will return list of an empty dict.
@@ -120,7 +123,7 @@ def make(
 
 
 class Strategy(BaseModel):
-    """Strategy Model that will combine a matrix together for running the
+    """Strategy model that will combine a matrix together for running the
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
@@ -162,11 +165,11 @@ class Strategy(BaseModel):
             "A matrix values that want to cross product to possible strategies."
         ),
     )
-    include: MatrixInclude = Field(
+    include: MatrixFilter = Field(
         default_factory=list,
         description="A list of additional matrix that want to adds-in.",
     )
-    exclude: MatrixExclude = Field(
+    exclude: MatrixFilter = Field(
         default_factory=list,
         description="A list of exclude matrix that want to filter-out.",
     )
@@ -200,12 +203,26 @@ class Strategy(BaseModel):
 
 
 class TriggerRules(str, Enum):
+    """Trigger rules enum object."""
+
     all_success: str = "all_success"
     all_failed: str = "all_failed"
+    all_done: str = "all_done"
+    one_failed: str = "one_failed"
+    one_success: str = "one_success"
+    none_failed: str = "none_failed"
+    none_skipped: str = "none_skipped"
+
+
+class RunsOn(str, Enum):
+    """Runs-On enum object."""
+
+    local: str = "local"
+    docker: str = "docker"
 
 
 class Job(BaseModel):
-    """Job Pydantic model object (group of stages).
+    """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
     you pass matrix mapping and it able to generate, you will see it running
@@ -264,12 +281,6 @@ class Job(BaseModel):
         default_factory=Strategy,
         description="A strategy matrix that want to generate.",
     )
-    run_id: Optional[str] = Field(
-        default=None,
-        description="A running job ID.",
-        repr=False,
-        exclude=True,
-    )
 
     @model_validator(mode="before")
     def __prepare_keys__(cls, values: DictData) -> DictData:
@@ -310,31 +321,22 @@ class Job(BaseModel):
         return value
 
     @model_validator(mode="after")
-    def __prepare_running_id_and_stage_name__(self) -> Self:
-        """Prepare the job running ID.
+    def __validate_job_id__(self) -> Self:
+        """Validate job id should not have templating syntax.
 
         :rtype: Self
         """
-        if self.run_id is None:
-            self.run_id = gen_id(self.id or "", unique=True)
-
         # VALIDATE: Validate job id should not dynamic with params template.
         if has_template(self.id):
             raise ValueError("Job ID should not has any template.")
 
         return self
 
-    def get_running_id(self, run_id: str) -> Self:
-        """Return Job model object that changing job running ID with an
-        input running ID.
-
-        :param run_id: A replace job running ID.
-        :rtype: Self
-        """
-        return self.model_copy(update={"run_id": run_id})
-
     def stage(self, stage_id: str) -> Stage:
-        """Return stage model that match with an input stage ID.
+        """Return stage instance that exists in this job via passing an input
+        stage ID.
+
+        :raise ValueError: If an input stage ID does not found on this job.
 
         :param stage_id: A stage ID that want to extract from this job.
         :rtype: Stage
@@ -367,8 +369,12 @@ class Job(BaseModel):
                         }
                     }
 
+        :raise JobException: If the job's ID does not set and the setting
+            default job ID flag does not set.
+
         :param output: An output context.
         :param to: A context data that want to add output result.
+
         :rtype: DictData
         """
         if self.id is None and not config.job_default_id:
@@ -383,8 +389,6 @@ class Job(BaseModel):
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
-
-        logger.debug(f"({self.run_id}) [JOB]: Set outputs on: {_id}")
         to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
@@ -397,6 +401,7 @@ class Job(BaseModel):
         strategy: DictData,
         params: DictData,
         *,
+        run_id: str | None = None,
         event: Event | None = None,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
@@ -406,15 +411,21 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
+            The result of this execution will return result with strategy ID
+        that generated from the `gen_id` function with a input strategy value.
+
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
 
-        :param strategy: A metrix strategy value.
-        :param params: A dynamic parameters.
+        :param strategy: A strategy metrix value that use on this execution.
+            This value will pass to the `matrix` key for templating.
+        :param params: A dynamic parameters that will deepcopy to the context.
+        :param run_id: A job running ID for this strategy execution.
         :param event: An manger event that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
+        run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
 
         # PARAGRAPH:
@@ -435,26 +446,23 @@ class Job(BaseModel):
         # IMPORTANT: The stage execution only run sequentially one-by-one.
         for stage in self.stages:
 
-            # IMPORTANT: Change any stage running IDs to this job running ID.
-            stage: Stage = stage.get_running_id(self.run_id)
-
-            name: str = stage.id or stage.name
-
             if stage.is_skipped(params=context):
-                logger.info(f"({self.run_id}) [JOB]: Skip stage: {name!r}")
+                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
                 continue
 
-            logger.info(
-                f"({self.run_id}) [JOB]: Start execute the stage: {name!r}"
-            )
+            logger.info(f"({run_id}) [JOB]: Execute stage: {stage.iden!r}")
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({self.run_id}) [JOB]: Matrix: {strategy}")
+                logger.info(f"({run_id}) [JOB]: ... Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
             if event and event.is_set():
+                error_msg: str = (
+                    "Job strategy was canceled from event that had set before "
+                    "strategy execution."
+                )
                 return Result(
                     status=1,
                     context={
@@ -464,17 +472,13 @@ class Job(BaseModel):
                             #   it will not filter function object from context.
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
+                            #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
-                            "error_message": (
-                                "Job strategy was canceled from trigger event "
-                                "that had stopped before execution."
-                            ),
+                            "error": JobException(error_msg),
+                            "error_message": error_msg,
                         },
                     },
+                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -497,12 +501,12 @@ class Job(BaseModel):
             #
             try:
                 stage.set_outputs(
-                    stage.execute(params=context).context,
+                    stage.execute(params=context, run_id=run_id).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
@@ -519,10 +523,10 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
+                    run_id=run_id,
                 )
 
-            # NOTE: Remove the current stage object that was created from
-            #   ``get_running_id`` method for saving memory.
+            # NOTE: Remove the current stage object for saving memory.
             del stage
 
         return Result(
@@ -533,28 +537,33 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
+            run_id=run_id,
         )
 
-    def execute(self, params: DictData | None = None) -> Result:
+    def execute(self, params: DictData, run_id: str | None = None) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the ``stages`` field of this job.
 
         :param params: An input parameters that use on job execution.
+        :param run_id: A job running ID for this execution.
+
         :rtype: Result
         """
 
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
-        params: DictData = {} if params is None else params
+        run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy.
+        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
                     strategy=strategy,
                     params=params,
+                    run_id=run_id,
                 )
                 context.update(rs.context)
             return Result(
@@ -572,41 +581,42 @@ class Job(BaseModel):
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
         ) as executor:
+
             futures: list[Future] = [
                 executor.submit(
                     self.execute_strategy,
                     strategy=strategy,
                     params=params,
+                    run_id=run_id,
                     event=event,
                 )
                 for strategy in self.strategy.make()
             ]
 
-            # NOTE: Dynamic catching futures object with fail-fast flag.
             return (
-                self.__catch_fail_fast(event=event, futures=futures)
+                self.__catch_fail_fast(event, futures=futures, run_id=run_id)
                 if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures)
+                else self.__catch_all_completed(futures=futures, run_id=run_id)
             )
 
+    @staticmethod
     def __catch_fail_fast(
-        self,
         event: Event,
         futures: list[Future],
+        run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop all not done futures if it receive the first exception from all
-        running futures.
+        stop and set event on all not done futures if it receive the first
+        exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the
-            observing thread/process.
+            observing multithreading.
         :param futures: A list of futures.
+        :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
         rs_final: Result = Result()
@@ -616,14 +626,12 @@ class Job(BaseModel):
         # NOTE: Get results from a collection of tasks with a timeout that has
         #   the first exception.
         done, not_done = wait(
-            futures,
-            timeout=timeout,
-            return_when=FIRST_EXCEPTION,
+            futures, timeout=timeout, return_when=FIRST_EXCEPTION
         )
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -636,11 +644,13 @@ class Job(BaseModel):
 
         future: Future
         for future in done:
+
+            # NOTE: Handle the first exception from feature
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({self.run_id}) [JOB]: One stage failed with: "
-                    f"{future.exception()}, shutting down this future."
+                    f"({run_id}) [JOB]: Fail-fast catching:\n\t"
+                    f"{future.exception()}"
                 )
                 context.update(
                     {
@@ -651,53 +661,37 @@ class Job(BaseModel):
                 continue
 
             # NOTE: Update the result context to main job context.
-            context.update(future.result(timeout=result_timeout).context)
+            context.update(future.result().context)
 
         return rs_final.catch(status=status, context=context)
 
+    @staticmethod
     def __catch_all_completed(
-        self,
         futures: list[Future],
+        run_id: str,
         *,
         timeout: int = 1800,
-        result_timeout: int = 60,
     ) -> Result:
         """Job parallel pool futures catching with all-completed mode.
 
-        :param futures: A list of futures that want to catch all completed
-            result.
+        :param futures: A list of futures.
+        :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
-        :param result_timeout: A timeout of getting result from the future
-            instance when it was running completely.
+
         :rtype: Result
         """
         rs_final: Result = Result()
         context: DictData = {}
         status: int = 0
+
         for future in as_completed(futures, timeout=timeout):
             try:
-                context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:  # pragma: no cov
-                status = 1
-                logger.warning(
-                    f"({self.run_id}) [JOB]: Task is hanging. Attempting to "
-                    f"kill."
-                )
-                future.cancel()
-                time.sleep(0.1)
-
-                stmt: str = (
-                    "Failed to cancel the task."
-                    if not future.cancelled()
-                    else "Task canceled successfully."
-                )
-                logger.warning(f"({self.run_id}) [JOB]: {stmt}")
+                context.update(future.result().context)
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({self.run_id}) [JOB]: Get stage exception with "
-                    f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
-                    f"{err}"
+                    f"({run_id}) [JOB]: All-completed catching:\n\t"
+                    f"{err.__class__.__name__}:\n\t{err}"
                 )
                 context.update(
                     {
@@ -705,4 +699,5 @@ class Job(BaseModel):
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
                 )
+
         return rs_final.catch(status=status, context=context)

ddeutil/workflow/on.py

@@ -184,24 +184,13 @@ class On(BaseModel):
             raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
-    def next(self, start: str | datetime) -> datetime:
+    def next(self, start: str | datetime) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
         """
-        return self.generate(start=start).next
-
-    # def pop(self, queue: list[datetime]) -> datetime:
-    #     """Pop the matching datetime value from list of datetime alias queue."""
-    #     for dt in queue:
-    #         if self.next(dt) == dt:
-    #             return dt
-    #
-    #     # NOTE: Add 1 second value to the current datetime for forcing crontab
-    #     #   runner generate the next datetime instead if current datetime be
-    #     #   valid because I already replaced second to zero before passing.
-    #     return datetime.now(tz=config.tz).replace(
-    #         second=0, microsecond=0
-    #     ) + timedelta(seconds=1)
+        runner: CronRunner = self.generate(start=start)
+        _ = runner.next
+        return runner
 
 
 class YearOn(On):