ddeutil-workflow 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl

ddeutil/workflow/pipeline.py

@@ -29,7 +29,12 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, MatrixExclude, MatrixInclude
-from .exceptions import JobException, PipelineException, StageException
+from .exceptions import (
+    JobException,
+    PipelineException,
+    StageException,
+    UtilException,
+)
 from .loader import Loader
 from .on import On
 from .scheduler import CronRunner
@@ -39,6 +44,7 @@ from .utils import (
     Result,
     cross_product,
     dash2underscore,
+    filter_func,
     gen_id,
     get_diff_sec,
 )
@@ -54,7 +60,7 @@ class Strategy(BaseModel):
         ...     'fail-fast': False,
         ...     'matrix': {
         ...         'first': [1, 2, 3],
-        ...         'second': ['foo', 'bar']
+        ...         'second': ['foo', 'bar'],
         ...     },
         ...     'include': [{'first': 4, 'second': 'foo'}],
         ...     'exclude': [{'first': 1, 'second': 'bar'}],
@@ -82,6 +88,10 @@ class Strategy(BaseModel):
         dash2underscore("fail-fast", values)
         return values
 
+    def is_set(self) -> bool:
+        """Return True if this strategy was set from yaml template."""
+        return len(self.matrix) > 0
+
     def make(self) -> list[DictStr]:
         """Return List of product of matrix values that already filter with
         exclude and add include.
@@ -138,18 +148,25 @@ class Job(BaseModel):
     Data Validate:
         >>> job = {
         ...     "runs-on": None,
-        ...     "strategy": {},
+        ...     "strategy": {
+        ...         "max-parallel": 1,
+        ...         "matrix": {
+        ...             "first": [1, 2, 3],
+        ...             "second": ['foo', 'bar'],
+        ...         },
+        ...     },
         ...     "needs": [],
         ...     "stages": [
         ...         {
         ...             "name": "Some stage",
         ...             "run": "print('Hello World')",
         ...         },
+        ...         ...
         ...     ],
         ... }
     """
 
-    name: Optional[str] = Field(default=None)
+    id: Optional[str] = Field(default=None)
     desc: Optional[str] = Field(default=None)
     runs_on: Optional[str] = Field(default=None)
     stages: list[Stage] = Field(
@@ -164,6 +181,9 @@ class Job(BaseModel):
         default_factory=Strategy,
         description="A strategy matrix that want to generate.",
     )
+    run_id: Optional[str] = Field(
+        default=None, description="A running job ID.", repr=False
+    )
 
     @model_validator(mode="before")
     def __prepare_keys(cls, values: DictData) -> DictData:
@@ -173,6 +193,12 @@ class Job(BaseModel):
         dash2underscore("runs-on", values)
         return values
 
+    @model_validator(mode="after")
+    def __prepare_running_id(self):
+        if self.run_id is None:
+            self.run_id = gen_id(self.id or "", unique=True)
+        return self
+
     def stage(self, stage_id: str) -> Stage:
         """Return stage model that match with an input stage ID."""
         for stage in self.stages:
@@ -180,9 +206,8 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
-    @staticmethod
-    def set_outputs(output: DictData) -> DictData:
-        if len(output) > 1:
+    def set_outputs(self, output: DictData) -> DictData:
+        if len(output) > 1 and self.strategy.is_set():
             return {"strategies": output}
 
         return output[next(iter(output))]
@@ -194,26 +219,32 @@ class Job(BaseModel):
         *,
         event: Event | None = None,
     ) -> Result:
-        """Strategy execution with passing dynamic parameters from the pipeline
-        stage execution.
+        """Job Strategy execution with passing dynamic parameters from the
+        pipeline execution to strategy matrix.
 
-        :param strategy:
-        :param params:
+            This execution is the minimum level execution of job model.
+
+        :param strategy: A metrix strategy value.
+        :param params: A dynamic parameters.
         :param event: An manger event that pass to the PoolThreadExecutor.
         :rtype: Result
+
+        :raise JobException: If it has any error from StageException or
+            UtilException.
         """
-        _stop_rs: Result = Result(
-            status=1,
-            context={
-                gen_id(strategy): {
-                    "matrix": strategy,
-                    "stages": {},
-                    "error": "Event stopped",
-                },
-            },
-        )
         if event and event.is_set():
-            return _stop_rs
+            return Result(
+                status=1,
+                context={
+                    gen_id(strategy): {
+                        "matrix": strategy,
+                        "stages": {},
+                        "error": {
+                            "message": "Process Event stopped before execution"
+                        },
+                    },
+                },
+            )
 
         # NOTE: Create strategy execution context and update a matrix and copied
         #   of params. So, the context value will have structure like;
@@ -229,16 +260,25 @@ class Job(BaseModel):
 
         # IMPORTANT: The stage execution only run sequentially one-by-one.
         for stage in self.stages:
+
+            # IMPORTANT: Change any stage running IDs to this job running ID.
+            stage.run_id = self.run_id
+
             _st_name: str = stage.id or stage.name
 
-            if stage.is_skip(params=context):
-                logging.info(f"[JOB]: Skip the stage: {_st_name!r}")
+            if stage.is_skipped(params=context):
+                logging.info(
+                    f"({self.run_id}) [JOB]: Skip the stage: {_st_name!r}"
+                )
                 continue
-            logging.info(f"[JOB]: Start execute the stage: {_st_name!r}")
+
+            logging.info(
+                f"({self.run_id}) [JOB]: Start execute the stage: {_st_name!r}"
+            )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logging.info(f"[...]: Matrix: {strategy}")
+                logging.info(f"({self.run_id}) [JOB]: Matrix: {strategy}")
 
             # NOTE:
             #       I do not use below syntax because `params` dict be the
@@ -258,23 +298,41 @@ class Job(BaseModel):
             #   }
             #
             if event and event.is_set():
-                return _stop_rs
-            rs: Result = stage.execute(params=context)
-            if rs.status == 0:
+                return Result(
+                    status=1,
+                    context={
+                        gen_id(strategy): {
+                            "matrix": strategy,
+                            "stages": filter_func(context.pop("stages", {})),
+                            "error": {
+                                "message": (
+                                    "Process Event stopped before execution"
+                                ),
+                            },
+                        },
+                    },
+                )
+            try:
+                rs: Result = stage.execute(params=context)
                 stage.set_outputs(rs.context, params=context)
-            else:
-                raise JobException(
-                    f"Getting status does not equal zero on stage: "
-                    f"{stage.name}."
+            except (StageException, UtilException) as err:
+                logging.error(
+                    f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
-        # TODO: Filter and warning if it pass any objects to context between
-        #   strategy job executor like function, etc.
+                raise JobException(
+                    f"Get stage execution error: {err.__class__.__name__}: "
+                    f"{err}"
+                ) from None
         return Result(
             status=0,
             context={
                 gen_id(strategy): {
                     "matrix": strategy,
-                    "stages": context.pop("stages", {}),
+                    # NOTE: (WF001) filter own created function from stages
+                    #   value, because it does not dump with pickle when you
+                    #   execute with multiprocess.
+                    #
+                    "stages": filter_func(context.pop("stages", {})),
                 },
             },
         )
@@ -288,17 +346,20 @@ class Job(BaseModel):
         :rtype: Result
         """
         strategy_context: DictData = {}
-        rs = Result(context=strategy_context)
 
-        if self.strategy.max_parallel == 1:
+        # NOTE: Normal Job execution.
+        if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.strategy_execute(
                     strategy, params=copy.deepcopy(params)
                 )
                 strategy_context.update(rs.context)
-            return rs
+            return Result(
+                status=0,
+                context=strategy_context,
+            )
 
-        # FIXME: (WF001) I got error that raise when use
+        # WARNING: (WF001) I got error that raise when use
         #  ``ProcessPoolExecutor``;
         #   ---
         #   _pickle.PicklingError: Can't pickle
@@ -308,84 +369,109 @@ class Job(BaseModel):
         with Manager() as manager:
             event: Event = manager.Event()
 
+            # NOTE: Start process pool executor for running strategy executor in
+            #   parallel mode.
             with ProcessPoolExecutor(
                 max_workers=self.strategy.max_parallel
-            ) as pool:
-                pool_result: list[Future] = [
-                    pool.submit(
+            ) as executor:
+                features: list[Future] = [
+                    executor.submit(
                         self.strategy_execute,
-                        st,
+                        strategy,
                         params=copy.deepcopy(params),
                         event=event,
                     )
-                    for st in self.strategy.make()
+                    for strategy in self.strategy.make()
                 ]
                 if self.strategy.fail_fast:
+                    rs = self.__catch_fail_fast(event, features)
+                else:
+                    rs = self.__catch_all_completed(features)
+        return Result(
+            status=0,
+            context=rs.context,
+        )
 
-                    # NOTE: Get results from a collection of tasks with a
-                    #   timeout that has the first exception.
-                    done, not_done = wait(
-                        pool_result, timeout=60, return_when=FIRST_EXCEPTION
-                    )
-                    nd: str = (
-                        f", the strategies do not run is {not_done}"
-                        if not_done
-                        else ""
-                    )
-                    logging.warning(f"[JOB]: Strategy is set Fail Fast{nd}")
-
-                    # NOTE: Stop all running tasks
-                    event.set()
-
-                    # NOTE: Cancel any scheduled tasks
-                    for future in pool_result:
-                        future.cancel()
-
-                    rs.status = 0
-                    for f in done:
-                        if f.exception():
-                            rs.status = 1
-                            logging.error(
-                                f"One task failed with: {f.exception()}, "
-                                f"shutting down"
-                            )
-                        elif f.cancelled():
-                            continue
-                        else:
-                            rs: Result = f.result(timeout=60)
-                            strategy_context.update(rs.context)
-                    rs.context = strategy_context
-                    return rs
-
-                for pool_rs in as_completed(pool_result):
-                    try:
-                        rs: Result = pool_rs.result(timeout=60)
-                        strategy_context.update(rs.context)
-                    except PickleError as err:
-                        # NOTE: I do not want to fix this issue because it does
-                        #   not make sense and over-engineering with this bug
-                        #   fix process.
-                        raise JobException(
-                            f"PyStage that create object on locals does use "
-                            f"parallel in strategy;\n\t{err}"
-                        ) from None
-                    except TimeoutError:
-                        rs.status = 1
-                        logging.warning("Task is hanging. Attempting to kill.")
-                        pool_rs.cancel()
-                        if not pool_rs.cancelled():
-                            logging.warning("Failed to cancel the task.")
-                        else:
-                            logging.warning("Task canceled successfully.")
-                    except StageException as err:
-                        rs.status = 1
-                        logging.warning(
-                            f"Get stage exception with fail-fast does not set;"
-                            f"\n\t{err}"
-                        )
-        rs.status = 0
-        rs.context = strategy_context
-        return rs
+    def __catch_fail_fast(self, event: Event, features: list[Future]) -> Result:
+        """Job parallel pool features catching with fail-fast mode. That will
+        stop all not done features if it receive the first exception from all
+        running features.
+
+        :param event:
+        :param features: A list of features.
+        :rtype: Result
+        """
+        strategy_context: DictData = {}
+        # NOTE: Get results from a collection of tasks with a
+        #   timeout that has the first exception.
+        done, not_done = wait(
+            features, timeout=1800, return_when=FIRST_EXCEPTION
+        )
+        nd: str = (
+            f", the strategies do not run is {not_done}" if not_done else ""
+        )
+        logging.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
+        # NOTE: Stop all running tasks
+        event.set()
+
+        # NOTE: Cancel any scheduled tasks
+        for future in features:
+            future.cancel()
+
+        status: int = 0
+        for f in done:
+            if f.exception():
+                status = 1
+                logging.error(
+                    f"({self.run_id}) [JOB]: One stage failed with: "
+                    f"{f.exception()}, shutting down this feature."
+                )
+            elif f.cancelled():
+                continue
+            else:
+                rs: Result = f.result(timeout=60)
+                strategy_context.update(rs.context)
+        return Result(
+            status=status,
+            context=strategy_context,
+        )
+
+    def __catch_all_completed(self, features: list[Future]) -> Result:
+        """Job parallel pool features catching with all-completed mode.
+
+        :param features: A list of features.
+        """
+        strategy_context: DictData = {}
+        status: int = 0
+        for feature in as_completed(features):
+            try:
+                rs: Result = feature.result(timeout=60)
+                strategy_context.update(rs.context)
+            except PickleError as err:
+                # NOTE: (WF001) I do not want to fix this issue because
+                #   it does not make sense and over-engineering with
+                #   this bug fix process.
+                raise JobException(
+                    f"PyStage that create object on locals does use "
+                    f"parallel in strategy execution;\n\t{err}"
+                ) from None
+            except TimeoutError:
+                status = 1
+                logging.warning("Task is hanging. Attempting to kill.")
+                feature.cancel()
+                if not feature.cancelled():
+                    logging.warning("Failed to cancel the task.")
+                else:
+                    logging.warning("Task canceled successfully.")
+            except JobException as err:
+                status = 1
+                logging.error(
+                    f"({self.run_id}) [JOB]: Get stage exception with "
+                    f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
+                    f"{err}"
+                )
+        return Result(status=status, context=strategy_context)
 
 
 class Pipeline(BaseModel):
@@ -414,6 +500,9 @@ class Pipeline(BaseModel):
         default_factory=dict,
         description="A mapping of job ID and job model that already loaded.",
     )
+    run_id: Optional[str] = Field(
+        default=None, description="A running job ID.", repr=False
+    )
 
     @classmethod
     def from_loader(
@@ -474,7 +563,7 @@ class Pipeline(BaseModel):
         return values
 
     @model_validator(mode="after")
-    def __validate_jobs_need(self):
+    def __validate_jobs_need_and_prepare_running_id(self):
         for job in self.jobs:
             if not_exist := [
                 need for need in self.jobs[job].needs if need not in self.jobs
@@ -483,6 +572,13 @@ class Pipeline(BaseModel):
                     f"This needed jobs: {not_exist} do not exist in this "
                     f"pipeline."
                 )
+
+            # NOTE: update a job id with its job id from pipeline template
+            self.jobs[job].id = job
+
+        if self.run_id is None:
+            self.run_id = gen_id(self.name, unique=True)
+
         return self
 
     def job(self, name: str) -> Job:
@@ -602,7 +698,7 @@ class Pipeline(BaseModel):
         self,
         job: str,
         params: DictData,
-    ):
+    ) -> Result:
         """Job Executor that use on pipeline executor.
         :param job: A job ID that want to execute.
         :param params: A params that was parameterized from pipeline execution.
@@ -613,18 +709,19 @@ class Pipeline(BaseModel):
                 f"The job ID: {job} does not exists on {self.name!r} pipeline."
             )
 
-        job_obj: Job = self.jobs[job]
-
-        rs: Result = job_obj.execute(params=params)
-        if rs.status != 0:
-            logging.warning(
-                f"Getting status does not equal zero on job: {job}."
-            )
-            return Result(
-                status=1, context={job: job_obj.set_outputs(rs.context)}
-            )
-
-        return Result(status=0, context={job: job_obj.set_outputs(rs.context)})
+        try:
+            logging.info(f"({self.run_id}) [PIPELINE]: Start execute: {job!r}")
+            job_obj: Job = self.jobs[job]
+            j_rs: Result = job_obj.execute(params=params)
+        except JobException as err:
+            raise PipelineException(
+                f"The job ID: {job} get raise error: {err.__class__.__name__}:"
+                f"\n{err}"
+            ) from None
+        return Result(
+            status=j_rs.status,
+            context={job: job_obj.set_outputs(j_rs.context)},
+        )
 
     def execute(
         self,
@@ -666,68 +763,131 @@ class Pipeline(BaseModel):
             logging.warning("[PIPELINE]: This pipeline does not have any jobs")
             return Result(status=0, context=params)
 
-        # NOTE: create a job queue that keep the job that want to running after
+        # NOTE: Create a job queue that keep the job that want to running after
         #   it dependency condition.
         jq: Queue = Queue()
         for job_id in self.jobs:
             jq.put(job_id)
 
+        # NOTE: Create start timestamp
         ts: float = time.monotonic()
-        not_time_out_flag: bool = True
 
         # NOTE: Create result context that will pass this context to any
         #   execution dependency.
         rs: Result = Result(context=self.parameterize(params))
-        if (
-            worker := int(os.getenv("WORKFLOW_CORE_MAX_JOB_PARALLEL", "1"))
-        ) > 1:
-            # IMPORTANT: The job execution can run parallel and waiting by
-            #   needed.
-            with ThreadPoolExecutor(max_workers=worker) as executor:
-                futures: list[Future] = []
-                while not jq.empty() and (
-                    not_time_out_flag := ((time.monotonic() - ts) < timeout)
-                ):
-                    job_id: str = jq.get()
-                    logging.info(
-                        f"[PIPELINE]: Start execute the job: {job_id!r}"
-                    )
-                    job: Job = self.jobs[job_id]
-                    if any(
-                        need not in rs.context["jobs"] for need in job.needs
-                    ):
-                        jq.put(job_id)
-                    futures.append(
-                        executor.submit(
-                            self.job_execute,
-                            job_id,
-                            params=copy.deepcopy(rs.context),
-                        ),
+        try:
+            rs.receive(
+                self.__exec_non_threading(rs, jq, ts, timeout=timeout)
+                if (
+                    worker := int(
+                        os.getenv("WORKFLOW_CORE_MAX_JOB_PARALLEL", "1")
                     )
-                for future in as_completed(futures):
-                    job_rs: Result = future.result(timeout=20)
-                    rs.context["jobs"].update(job_rs.context)
-        else:
-            logging.info(
-                f"[CORE]: Run {self.name} with non-threading job executor"
+                )
+                == 1
+                else self.__exec_threading(
+                    rs, jq, ts, worker=worker, timeout=timeout
+                )
             )
-            while not jq.empty() and (
+            return rs
+        except PipelineException as err:
+            rs.context.update({"error": {"message": str(err)}})
+            rs.status = 1
+            return rs
+
+    def __exec_threading(
+        self,
+        rs: Result,
+        job_queue: Queue,
+        ts: float,
+        *,
+        worker: int = 1,
+        timeout: int = 600,
+    ) -> Result:
+        """Pipeline threading execution."""
+        not_time_out_flag: bool = True
+
+        # IMPORTANT: The job execution can run parallel and waiting by
+        #   needed.
+        with ThreadPoolExecutor(max_workers=worker) as executor:
+            futures: list[Future] = []
+            while not job_queue.empty() and (
                 not_time_out_flag := ((time.monotonic() - ts) < timeout)
             ):
-                job_id: str = jq.get()
-                logging.info(f"[PIPELINE]: Start execute the job: {job_id!r}")
+                job_id: str = job_queue.get()
                 job: Job = self.jobs[job_id]
-                if any(need not in rs.context["jobs"] for need in job.needs):
-                    jq.put(job_id)
 
-                job_rs = self.job_execute(
-                    job_id, params=copy.deepcopy(rs.context)
+                # IMPORTANT:
+                #   Change any job running IDs to this pipeline running ID.
+                job.run_id = self.run_id
+
+                if any(need not in rs.context["jobs"] for need in job.needs):
+                    job_queue.put(job_id)
+                    time.sleep(0.5)
+                    continue
+
+                futures.append(
+                    executor.submit(
+                        self.job_execute,
+                        job_id,
+                        params=copy.deepcopy(rs.context),
+                    ),
                 )
-                rs.context["jobs"].update(job_rs.context)
+
+            for future in as_completed(futures):
+                if err := future.exception():
+                    logging.error(f"{err}")
+                    raise PipelineException(f"{err}")
+
+                # NOTE: Update job result to pipeline result.
+                rs.receive_jobs(future.result(timeout=20))
 
         if not not_time_out_flag:
-            logging.warning("Execution of pipeline was time out")
-            rs.status = 1
-            return rs
+            logging.warning(
+                f"({self.run_id}) [PIPELINE]: Execution of pipeline was timeout"
+            )
+            raise PipelineException(
+                f"Execution of pipeline: {self.name} was timeout"
+            )
+        rs.status = 0
+        return rs
+
+    def __exec_non_threading(
+        self,
+        rs: Result,
+        job_queue: Queue,
+        ts: float,
+        *,
+        timeout: int = 600,
+    ) -> Result:
+        """Pipeline non-threading execution."""
+        not_time_out_flag: bool = True
+        logging.info(f"[CORE]: Run {self.name} with non-threading job executor")
+        while not job_queue.empty() and (
+            not_time_out_flag := ((time.monotonic() - ts) < timeout)
+        ):
+            job_id: str = job_queue.get()
+            job: Job = self.jobs[job_id]
+
+            # IMPORTANT:
+            #   Change any job running IDs to this pipeline running ID.
+            job.run_id = self.run_id
+
+            # NOTE:
+            if any(need not in rs.context["jobs"] for need in job.needs):
+                job_queue.put(job_id)
+                time.sleep(0.5)
+                continue
+
+            # NOTE: Start job execution.
+            job_rs = self.job_execute(job_id, params=copy.deepcopy(rs.context))
+            rs.context["jobs"].update(job_rs.context)
+
+        if not not_time_out_flag:
+            logging.warning(
+                f"({self.run_id}) [PIPELINE]: Execution of pipeline was timeout"
+            )
+            raise PipelineException(
+                f"Execution of pipeline: {self.name} was timeout"
+            )
         rs.status = 0
         return rs