ddeutil-workflow 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

ddeutil/workflow/job.py

@@ -3,6 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Job Model that use for keeping stages and node that running its stages.
+"""
 from __future__ import annotations
 
 import copy
@@ -14,11 +16,12 @@ from concurrent.futures import (
     as_completed,
     wait,
 )
-from pickle import PickleError
+from functools import lru_cache
 from textwrap import dedent
 from threading import Event
 from typing import Optional
 
+from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
@@ -53,12 +56,79 @@ logger = get_logger("ddeutil.workflow")
 __all__: TupleStr = (
     "Strategy",
     "Job",
+    "make",
 )
 
 
+@freeze_args
+@lru_cache
+def make(
+    matrix: Matrix,
+    include: MatrixInclude,
+    exclude: MatrixExclude,
+) -> list[DictStr]:
+    """Make a list of product of matrix values that already filter with
+    exclude matrix and add specific matrix with include.
+
+    :param matrix: A matrix values that want to cross product to possible
+        parallelism values.
+    :param include: A list of additional matrix that want to adds-in.
+    :param exclude: A list of exclude matrix that want to filter-out.
+    :rtype: list[DictStr]
+    """
+    # NOTE: If it does not set matrix, it will return list of an empty dict.
+    if len(matrix) == 0:
+        return [{}]
+
+    # NOTE: Remove matrix that exists on the exclude.
+    final: list[DictStr] = []
+    for r in cross_product(matrix=matrix):
+        if any(
+            all(r[k] == v for k, v in exclude.items()) for exclude in exclude
+        ):
+            continue
+        final.append(r)
+
+    # NOTE: If it is empty matrix and include, it will return list of an
+    #   empty dict.
+    if len(final) == 0 and not include:
+        return [{}]
+
+    # NOTE: Add include to generated matrix with exclude list.
+    add: list[DictStr] = []
+    for inc in include:
+        # VALIDATE:
+        #   Validate any key in include list should be a subset of some one
+        #   in matrix.
+        if all(not (set(inc.keys()) <= set(m.keys())) for m in final):
+            raise ValueError(
+                "Include should have the keys that equal to all final matrix."
+            )
+
+        # VALIDATE:
+        #   Validate value of include should not duplicate with generated
+        #   matrix. So, it will skip if this value already exists.
+        if any(
+            all(inc.get(k) == v for k, v in m.items()) for m in [*final, *add]
+        ):
+            continue
+        add.append(inc)
+
+    # NOTE: Merge all matrix together.
+    final.extend(add)
+    return final
+
+
 class Strategy(BaseModel):
     """Strategy Model that will combine a matrix together for running the
-    special job.
+    special job with combination of matrix data.
+
+        This model does not be the part of job only because you can use it to
+    any model object. The propose of this model is generate metrix result that
+    comming from combination logic with any matrix values for running it with
+    parallelism.
+
+        [1, 2, 3] x [a, b] --> [1a], [1b], [2a], [2b], [3a], [3b]
 
     Data Validate:
         >>> strategy = {
@@ -105,13 +175,19 @@ class Strategy(BaseModel):
     def __prepare_keys(cls, values: DictData) -> DictData:
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
+
+        :param values: A parsing values to this models
+        :rtype: DictData
         """
         dash2underscore("max-parallel", values)
         dash2underscore("fail-fast", values)
         return values
 
     def is_set(self) -> bool:
-        """Return True if this strategy was set from yaml template."""
+        """Return True if this strategy was set from yaml template.
+
+        :rtype: bool
+        """
         return len(self.matrix) > 0
 
     def make(self) -> list[DictStr]:
@@ -120,48 +196,11 @@ class Strategy(BaseModel):
 
         :rtype: list[DictStr]
         """
-        # NOTE: If it does not set matrix, it will return list of an empty dict.
-        if not (mt := self.matrix):
-            return [{}]
-
-        final: list[DictStr] = []
-        for r in cross_product(matrix=mt):
-            if any(
-                all(r[k] == v for k, v in exclude.items())
-                for exclude in self.exclude
-            ):
-                continue
-            final.append(r)
-
-        # NOTE: If it is empty matrix and include, it will return list of an
-        #   empty dict.
-        if not final and not self.include:
-            return [{}]
-
-        # NOTE: Add include to generated matrix with exclude list.
-        add: list[DictStr] = []
-        for include in self.include:
-            # VALIDATE:
-            #   Validate any key in include list should be a subset of some one
-            #   in matrix.
-            if all(not (set(include.keys()) <= set(m.keys())) for m in final):
-                raise ValueError("Include should have the keys equal to matrix")
-
-            # VALIDATE:
-            #   Validate value of include does not duplicate with generated
-            #   matrix.
-            if any(
-                all(include.get(k) == v for k, v in m.items())
-                for m in [*final, *add]
-            ):
-                continue
-            add.append(include)
-        final.extend(add)
-        return final
+        return make(self.matrix, self.include, self.exclude)
 
 
 class Job(BaseModel):
-    """Job Model (group of stages).
+    """Job Pydantic model object (group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
     you pass matrix mapping and it able to generate, you will see it running
@@ -191,8 +230,7 @@ class Job(BaseModel):
     id: Optional[str] = Field(
         default=None,
         description=(
-            "A job ID, this value will add from workflow after validation "
-            "process."
+            "A job ID that it will add from workflow after validation process."
         ),
     )
     desc: Optional[str] = Field(
@@ -227,6 +265,9 @@ class Job(BaseModel):
     def __prepare_keys(cls, values: DictData) -> DictData:
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
+
+        :param values: A passing value that coming for initialize this object.
+        :rtype: DictData
         """
         dash2underscore("runs-on", values)
         return values
@@ -237,7 +278,11 @@ class Job(BaseModel):
         return dedent(value)
 
     @model_validator(mode="after")
-    def __prepare_running_id(self):
+    def __prepare_running_id(self) -> Self:
+        """Prepare the job running ID.
+
+        :rtype: Self
+        """
         if self.run_id is None:
             self.run_id = gen_id(self.id or "", unique=True)
 
@@ -257,17 +302,51 @@ class Job(BaseModel):
         return self.model_copy(update={"run_id": run_id})
 
     def stage(self, stage_id: str) -> Stage:
-        """Return stage model that match with an input stage ID."""
+        """Return stage model that match with an input stage ID.
+
+        :param stage_id: A stage ID that want to extract from this job.
+        :rtype: Stage
+        """
         for stage in self.stages:
             if stage_id == (stage.id or ""):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
-    def set_outputs(self, output: DictData) -> DictData:
-        """Setting output of job execution"""
-        if len(output) > 1 and self.strategy.is_set():
-            return {"strategies": output}
-        return output[next(iter(output))]
+    def set_outputs(self, output: DictData, to: DictData) -> DictData:
+        """Set an outputs from execution process to the receive context. The
+        result from execution will pass to value of ``strategies`` key.
+
+            For example of setting output method, If you receive execute output
+        and want to set on the `to` like;
+
+            ... (i)   output: {'strategy01': bar, 'strategy02': bar}
+            ... (ii)  to: {}
+
+        The result of the `to` variable will be;
+
+            ... (iii) to: {
+                            'strategies': {
+                                'strategy01': bar, 'strategy02': bar
+                            }
+                        }
+
+        :param output: An output context.
+        :param to: A context data that want to add output result.
+        :rtype: DictData
+        """
+        if self.id is None:
+            raise JobException(
+                "This job do not set the ID before setting output."
+            )
+
+        to[self.id] = (
+            {"strategies": output}
+            if self.strategy.is_set()
+            # NOTE:
+            #   This is the best way to get single key from dict.
+            else output[next(iter(output))]
+        )
+        return to
 
     def execute_strategy(
         self,
@@ -275,46 +354,38 @@ class Job(BaseModel):
         params: DictData,
         *,
         event: Event | None = None,
+        raise_error: bool = True,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
         workflow execution to strategy matrix.
 
-            This execution is the minimum level execution of job model.
+            This execution is the minimum level of execution of this job model.
+        It different with ``self.execute`` because this method run only one
+        strategy and return with context of this strategy data.
+
+        :raise JobException: If it has any error from StageException or
+            UtilException.
 
         :param strategy: A metrix strategy value.
         :param params: A dynamic parameters.
         :param event: An manger event that pass to the PoolThreadExecutor.
+        :param raise_error: A flag that raise error instead catching to result
+            if it get exception from stage execution.
         :rtype: Result
-
-        :raise JobException: If it has any error from StageException or
-            UtilException.
         """
-        # NOTE: Force stop this execution if event was set from main execution.
-        if event and event.is_set():
-            return Result(
-                status=1,
-                context={
-                    gen_id(strategy): {
-                        "matrix": strategy,
-                        "stages": {},
-                        "error_message": {
-                            "message": "Process Event stopped before execution"
-                        },
-                    },
-                },
-            )
+        strategy_id: str = gen_id(strategy)
 
         # NOTE: Create strategy execution context and update a matrix and copied
         #   of params. So, the context value will have structure like;
-        #   ---
+        #
         #   {
         #       "params": { ... },      <== Current input params
         #       "jobs": { ... },        <== Current input params
         #       "matrix": { ... }       <== Current strategy value
         #   }
         #
-        context: DictData = params
-        context.update({"matrix": strategy})
+        context: DictData = copy.deepcopy(params)
+        context.update({"matrix": strategy, "stages": {}})
 
         # IMPORTANT: The stage execution only run sequentially one-by-one.
         for stage in self.stages:
@@ -325,9 +396,7 @@ class Job(BaseModel):
             _st_name: str = stage.id or stage.name
 
             if stage.is_skipped(params=context):
-                logger.info(
-                    f"({self.run_id}) [JOB]: Skip the stage: {_st_name!r}"
-                )
+                logger.info(f"({self.run_id}) [JOB]: Skip stage: {_st_name!r}")
                 continue
 
             logger.info(
@@ -338,34 +407,23 @@ class Job(BaseModel):
             if strategy:
                 logger.info(f"({self.run_id}) [JOB]: Matrix: {strategy}")
 
-            # NOTE:
-            #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer and it was changed when I action
-            #   anything like update or re-construct this.
-            #
-            #       ... params |= stage.execute(params=params)
-            #
-            #   This step will add the stage result to ``stages`` key in
-            #   that stage id. It will have structure like;
-            #   ---
-            #   {
-            #       "params": { ... },
-            #       "jobs": { ... },
-            #       "matrix": { ... },
-            #       "stages": { { "stage-id-1": ... }, ... }
-            #   }
-            #
+            # NOTE: Force stop this execution if event was set from main
+            #   execution.
             if event and event.is_set():
                 return Result(
                     status=1,
                     context={
-                        gen_id(strategy): {
+                        strategy_id: {
                             "matrix": strategy,
                             # NOTE: If job strategy executor use multithreading,
                             #   it will not filter function object from context.
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
                             "stages": context.pop("stages", {}),
+                            # NOTE: Set the error keys.
+                            "error": JobException(
+                                "Process Event stopped before execution"
+                            ),
                             "error_message": {
                                 "message": (
                                     "Process Event stopped before execution"
@@ -374,17 +432,40 @@ class Job(BaseModel):
                         },
                     },
                 )
+
+            # NOTE:
+            #       I do not use below syntax because `params` dict be the
+            #   reference memory pointer and it was changed when I action
+            #   anything like update or re-construct this.
+            #
+            #       ... params |= stage.execute(params=params)
+            #
+            #   This step will add the stage result to ``stages`` key in
+            #   that stage id. It will have structure like;
+            #
+            #   {
+            #       "params": { ... },
+            #       "jobs": { ... },
+            #       "matrix": { ... },
+            #       "stages": { { "stage-id-1": ... }, ... }
+            #   }
+            #
             try:
-                rs: Result = stage.execute(params=context)
-                stage.set_outputs(rs.context, to=context)
+                stage.set_outputs(
+                    stage.execute(params=context).context,
+                    to=context,
+                )
             except (StageException, UtilException) as err:
                 logger.error(
                     f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
-                raise JobException(
-                    f"Get stage execution error: {err.__class__.__name__}: "
-                    f"{err}"
-                ) from None
+                if raise_error:
+                    raise JobException(
+                        f"Get stage execution error: {err.__class__.__name__}: "
+                        f"{err}"
+                    ) from None
+                else:
+                    raise NotImplementedError() from None
 
             # NOTE: Remove new stage object that was created from
             #   ``get_running_id`` method.
@@ -393,12 +474,8 @@ class Job(BaseModel):
         return Result(
             status=0,
             context={
-                gen_id(strategy): {
+                strategy_id: {
                     "matrix": strategy,
-                    # NOTE: (WF001) filter own created function from stages
-                    #   value, because it does not dump with pickle when you
-                    #   execute with multiprocess.
-                    #
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
@@ -406,19 +483,21 @@ class Job(BaseModel):
 
     def execute(self, params: DictData | None = None) -> Result:
         """Job execution with passing dynamic parameters from the workflow
-        execution. It will generate matrix values at the first step and for-loop
-        any metrix to all stages dependency.
+        execution. It will generate matrix values at the first step and run
+        multithread on this metrics to the ``stages`` field of this job.
 
         :param params: An input parameters that use on job execution.
         :rtype: Result
         """
         context: DictData = {}
+        params: DictData = {} if params is None else params
 
-        # NOTE: Normal Job execution.
+        # NOTE: Normal Job execution without parallel strategy.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
-                    strategy, params=copy.deepcopy(params)
+                    strategy=strategy,
+                    params=copy.deepcopy(params),
                 )
                 context.update(rs.context)
             return Result(
@@ -426,36 +505,6 @@ class Job(BaseModel):
                 context=context,
             )
 
-        # # WARNING: (WF001) I got error that raise when use
-        # #  ``ProcessPoolExecutor``;
-        # #   ---
-        # #   _pickle.PicklingError: Can't pickle
-        # #       <function ??? at 0x000001F0BE80F160>: attribute lookup ???
-        # #       on ddeutil.workflow.stage failed
-        # #
-        # # from multiprocessing import Event, Manager
-        # with Manager() as manager:
-        #     event: Event = manager.Event()
-        #
-        #     # NOTE: Start process pool executor for running strategy executor
-        #     #   in parallel mode.
-        #     with ProcessPoolExecutor(
-        #         max_workers=self.strategy.max_parallel
-        #     ) as executor:
-        #         futures: list[Future] = [
-        #             executor.submit(
-        #                 self.execute_strategy,
-        #                 strategy,
-        #                 params=copy.deepcopy(params),
-        #                 event=event,
-        #             )
-        #             for strategy in self.strategy.make()
-        #         ]
-        #         if self.strategy.fail_fast:
-        #             rs = self.__catch_fail_fast(event, futures)
-        #         else:
-        #             rs = self.__catch_all_completed(futures)
-
         # NOTE: Create event for cancel executor stop running.
         event: Event = Event()
 
@@ -465,53 +514,65 @@ class Job(BaseModel):
             futures: list[Future] = [
                 executor.submit(
                     self.execute_strategy,
-                    strategy,
-                    params=copy.deepcopy(params),
+                    strategy=strategy,
+                    params=params,
                     event=event,
                 )
                 for strategy in self.strategy.make()
             ]
 
             # NOTE: Dynamic catching futures object with fail-fast flag.
-            if self.strategy.fail_fast:
-                rs: Result = self.__catch_fail_fast(event, futures)
-            else:
-                rs: Result = self.__catch_all_completed(futures)
-        return Result(
-            status=0,
-            context=rs.context,
-        )
+            return (
+                self.__catch_fail_fast(event=event, futures=futures)
+                if self.strategy.fail_fast
+                else self.__catch_all_completed(futures=futures)
+            )
 
-    def __catch_fail_fast(self, event: Event, futures: list[Future]) -> Result:
+    def __catch_fail_fast(
+        self,
+        event: Event,
+        futures: list[Future],
+        *,
+        timeout: int = 1800,
+        result_timeout: int = 60,
+    ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
         stop all not done futures if it receive the first exception from all
         running futures.
 
-        :param event:
+        :param event: An event manager instance that able to set stopper on the
+            observing thread/process.
         :param futures: A list of futures.
+        :param timeout: A timeout to waiting all futures complete.
+        :param result_timeout: A timeout of getting result from the future
+            instance when it was running completely.
         :rtype: Result
         """
+        rs_final: Result = Result()
         context: DictData = {}
-        # NOTE: Get results from a collection of tasks with a
-        #   timeout that has the first exception.
+        status: int = 0
+
+        # NOTE: Get results from a collection of tasks with a timeout that has
+        #   the first exception.
         done, not_done = wait(
-            futures, timeout=1800, return_when=FIRST_EXCEPTION
+            futures,
+            timeout=timeout,
+            return_when=FIRST_EXCEPTION,
         )
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
         logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
+        # NOTE: Stop all running tasks with setting the event manager and cancel
+        #   any scheduled tasks.
         if len(done) != len(futures):
-
-            # NOTE: Stop all running tasks
             event.set()
-
-            # NOTE: Cancel any scheduled tasks
             for future in futures:
                 future.cancel()
 
-        status: int = 0
+            del future
+
         for future in done:
             if future.exception():
                 status = 1
@@ -521,31 +582,36 @@ class Job(BaseModel):
                 )
             elif future.cancelled():
                 continue
-            else:
-                rs: Result = future.result(timeout=60)
-                context.update(rs.context)
-        return Result(status=status, context=context)
 
-    def __catch_all_completed(self, futures: list[Future]) -> Result:
+            # NOTE: Update the result context to main job context.
+            context.update(future.result(timeout=result_timeout).context)
+
+            del future
+
+        return rs_final.catch(status=status, context=context)
+
+    def __catch_all_completed(
+        self,
+        futures: list[Future],
+        *,
+        timeout: int = 1800,
+        result_timeout: int = 60,
+    ) -> Result:
         """Job parallel pool futures catching with all-completed mode.
 
-        :param futures: A list of futures.
+        :param futures: A list of futures that want to catch all completed
+            result.
+        :param timeout: A timeout to waiting all futures complete.
+        :param result_timeout: A timeout of getting result from the future
+            instance when it was running completely.
         :rtype: Result
         """
+        rs_final: Result = Result()
         context: DictData = {}
         status: int = 0
-        for future in as_completed(futures):
+        for future in as_completed(futures, timeout=timeout):
             try:
-                rs: Result = future.result(timeout=60)
-                context.update(rs.context)
-            except PickleError as err:
-                # NOTE: (WF001) I do not want to fix this issue because
-                #   it does not make sense and over-engineering with
-                #   this bug fix process.
-                raise JobException(
-                    f"PyStage that create object on locals does use "
-                    f"parallel in strategy execution;\n\t{err}"
-                ) from None
+                context.update(future.result(timeout=result_timeout).context)
             except TimeoutError:
                 status = 1
                 logger.warning(
@@ -554,14 +620,13 @@ class Job(BaseModel):
                 )
                 future.cancel()
                 time.sleep(0.1)
-                if not future.cancelled():
-                    logger.warning(
-                        f"({self.run_id}) [JOB]: Failed to cancel the task."
-                    )
-                else:
-                    logger.warning(
-                        f"({self.run_id}) [JOB]: Task canceled successfully."
-                    )
+
+                stmt: str = (
+                    "Failed to cancel the task."
+                    if not future.cancelled()
+                    else "Task canceled successfully."
+                )
+                logger.warning(f"({self.run_id}) [JOB]: {stmt}")
             except JobException as err:
                 status = 1
                 logger.error(
@@ -569,4 +634,6 @@ class Job(BaseModel):
                     f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
                     f"{err}"
                 )
-        return Result(status=status, context=context)
+            finally:
+                del future
+        return rs_final.catch(status=status, context=context)