ddeutil-workflow 0.0.57__py3-none-any.whl → 0.0.59__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -141,8 +141,8 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
@@ -162,12 +162,12 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-        event: Event | None = None,
-        raise_error: bool | None = None,
-    ) -> Result | DictData:
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+        raise_error: Optional[bool] = None,
+    ) -> Union[Result, DictData]:
         """Handler stage execution result from the stage `execute` method.
 
             This stage exception handler still use ok-error concept, but it
@@ -376,8 +376,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         raise NotImplementedError(
             "Async Stage should implement `execute` method."
@@ -388,8 +388,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -411,11 +411,11 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-        event: Event | None = None,
-        raise_error: bool | None = None,
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+        raise_error: Optional[bool] = None,
     ) -> Result:
         """Async Handler stage execution result from the stage `execute` method.
 
@@ -487,8 +487,8 @@ class EmptyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout.
@@ -507,13 +507,13 @@ class EmptyStage(BaseAsyncStage):
             run_id=gen_id(self.name + (self.id or ""), unique=True),
             extras=self.extras,
         )
-
-        if not self.echo:
-            message: str = "..."
-        else:
-            message: str = param2template(
+        message: str = (
+            param2template(
                 dedent(self.echo.strip("\n")), params, extras=self.extras
             )
+            if self.echo
+            else "..."
+        )
 
         result.trace.info(
             f"[STAGE]: Execute Empty-Stage: {self.name!r}: ( {message} )"
@@ -528,8 +528,8 @@ class EmptyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -546,12 +546,13 @@ class EmptyStage(BaseAsyncStage):
             extras=self.extras,
         )
 
-        if not self.echo:
-            message: str = "..."
-        else:
-            message: str = param2template(
+        message: str = (
+            param2template(
                 dedent(self.echo.strip("\n")), params, extras=self.extras
             )
+            if self.echo
+            else "..."
+        )
 
         result.trace.info(f"[STAGE]: Empty-Stage: {self.name!r}: ( {message} )")
         if self.sleep > 0:
@@ -598,13 +599,13 @@ class BashStage(BaseAsyncStage):
 
     @contextlib.asynccontextmanager
     async def acreate_sh_file(
-        self, bash: str, env: DictStr, run_id: str | None = None
+        self, bash: str, env: DictStr, run_id: Optional[str] = None
     ) -> AsyncIterator[TupleStr]:
         """Async create and write `.sh` file with the `aiofiles` package.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (str | None) A running stage ID that use for writing sh
+        :param run_id: (Optional[str]) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: AsyncIterator[TupleStr]
@@ -634,14 +635,14 @@ class BashStage(BaseAsyncStage):
 
     @contextlib.contextmanager
     def create_sh_file(
-        self, bash: str, env: DictStr, run_id: str | None = None
+        self, bash: str, env: DictStr, run_id: Optional[str] = None
     ) -> Iterator[TupleStr]:
         """Create and write the `.sh` file before giving this file name to
         context. After that, it will auto delete this file automatic.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (str | None) A running stage ID that use for writing sh
+        :param run_id: (Optional[str]) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: Iterator[TupleStr]
@@ -672,8 +673,8 @@ class BashStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute bash statement with the Python build-in `subprocess` package.
         It will catch result from the `subprocess.run` returning output like
@@ -729,8 +730,8 @@ class BashStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -857,8 +858,8 @@ class PyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on `exec` build-in function.
@@ -915,8 +916,8 @@ class PyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -1017,8 +1018,8 @@ class CallStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute this caller function with its argument parameter.
 
@@ -1107,8 +1108,8 @@ class CallStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -1265,8 +1266,8 @@ class TriggerStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
@@ -1297,18 +1298,49 @@ class TriggerStage(BaseStage):
             event=event,
         )
         if rs.status == FAILED:
-            err_msg: str | None = (
+            err_msg: Optional[str] = (
                 f" with:\n{msg}"
                 if (msg := rs.context.get("errors", {}).get("message"))
                 else "."
             )
             raise StageException(
-                f"Trigger workflow return failed status{err_msg}"
+                f"Trigger workflow return `FAILED` status{err_msg}"
             )
         return rs
 
 
-class ParallelStage(BaseStage):
+class BaseNestedStage(BaseStage):
+    """Base Nested Stage model. This model is use for checking the child stage
+    is the nested stage or not.
+    """
+
+    @abstractmethod
+    def execute(
+        self,
+        params: DictData,
+        *,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+    ) -> Result:
+        """Execute abstraction method that action something by sub-model class.
+        This is important method that make this class is able to be the nested
+        stage.
+
+        :param params: (DictData) A parameter data that want to use in this
+            execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        raise NotImplementedError(
+            "Nested-Stage should implement `execute` method."
+        )
+
+
+class ParallelStage(BaseNestedStage):
     """Parallel stage executor that execute branch stages with multithreading.
     This stage let you set the fix branches for running child stage inside it on
     multithread pool.
@@ -1358,7 +1390,7 @@ class ParallelStage(BaseStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute all stage with specific branch ID.
 
@@ -1455,8 +1487,8 @@ class ParallelStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute parallel each branch via multi-threading pool.
 
@@ -1494,7 +1526,7 @@ class ParallelStage(BaseStage):
             context: DictData = {}
             status: Status = SUCCESS
 
-            futures: list[Future] = (
+            futures: list[Future] = [
                 executor.submit(
                     self.execute_branch,
                     branch=branch,
@@ -1503,26 +1535,21 @@ class ParallelStage(BaseStage):
                     event=event,
                 )
                 for branch in self.parallel
-            )
+            ]
 
             for future in as_completed(futures):
                 try:
                     future.result()
                 except StageException as e:
                     status = FAILED
-                    result.trace.error(
-                        f"[STAGE]: Error Handler:||{e.__class__.__name__}:||{e}"
-                    )
                     if "errors" in context:
                         context["errors"][e.refs] = e.to_dict()
                     else:
                         context["errors"] = e.to_dict(with_refs=True)
-                except CancelledError:
-                    pass
         return result.catch(status=status, context=context)
 
 
-class ForEachStage(BaseStage):
+class ForEachStage(BaseNestedStage):
     """For-Each stage executor that execute all stages with each item in the
     foreach list.
 
@@ -1563,18 +1590,27 @@ class ForEachStage(BaseStage):
             "will be sequential mode if this value equal 1."
         ),
     )
+    use_index_as_key: bool = Field(
+        default=False,
+        description=(
+            "A flag for using the loop index as a key instead item value. "
+            "This flag allow to skip checking duplicate item step."
+        ),
+    )
 
     def execute_item(
         self,
+        index: int,
         item: StrOrInt,
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute all nested stage that set on this stage with specific foreach
         item parameter.
 
+        :param index: (int) An index value of foreach loop.
         :param item: (str | int) An item that want to execution.
         :param params: (DictData) A parameter data.
         :param result: (Result) A Result instance for return context and status.
@@ -1588,8 +1624,9 @@ class ForEachStage(BaseStage):
         :rtype: Result
         """
         result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
+        key: StrOrInt = index if self.use_index_as_key else item
         context: DictData = copy.deepcopy(params)
-        context.update({"item": item})
+        context.update({"item": item, "loop": index})
         output: DictData = {"item": item, "stages": {}}
         for stage in self.stages:
 
@@ -1608,14 +1645,14 @@ class ForEachStage(BaseStage):
                 result.catch(
                     status=CANCEL,
                     foreach={
-                        item: {
+                        key: {
                             "item": item,
                             "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         }
                     },
                 )
-                raise StageException(error_msg, refs=item)
+                raise StageException(error_msg, refs=key)
 
             try:
                 rs: Result = stage.handler_execute(
@@ -1631,14 +1668,14 @@ class ForEachStage(BaseStage):
                 result.catch(
                     status=FAILED,
                     foreach={
-                        item: {
+                        key: {
                             "item": item,
                             "stages": filter_func(output.pop("stages", {})),
                             "errors": e.to_dict(),
                         },
                     },
                 )
-                raise StageException(str(e), refs=item) from e
+                raise StageException(str(e), refs=key) from e
 
             if rs.status == FAILED:
                 error_msg: str = (
@@ -1649,19 +1686,19 @@ class ForEachStage(BaseStage):
                 result.catch(
                     status=FAILED,
                     foreach={
-                        item: {
+                        key: {
                             "item": item,
                             "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         },
                     },
                 )
-                raise StageException(error_msg, refs=item)
+                raise StageException(error_msg, refs=key)
 
         return result.catch(
             status=SUCCESS,
             foreach={
-                item: {
+                key: {
                     "item": item,
                     "stages": filter_func(output.pop("stages", {})),
                 },
@@ -1672,8 +1709,8 @@ class ForEachStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the stages that pass each item form the foreach field.
 
@@ -1700,6 +1737,11 @@ class ForEachStage(BaseStage):
         # [VALIDATE]: Type of the foreach should be `list` type.
         if not isinstance(foreach, list):
             raise TypeError(f"Does not support foreach: {foreach!r}")
+        elif len(set(foreach)) != len(foreach) and not self.use_index_as_key:
+            raise ValueError(
+                "Foreach item should not duplicate. If this stage must to pass "
+                "duplicate item, it should set `use_index_as_key: true`."
+            )
 
         result.trace.info(f"[STAGE]: Execute Foreach-Stage: {foreach!r}.")
         result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
@@ -1721,18 +1763,19 @@ class ForEachStage(BaseStage):
             futures: list[Future] = [
                 executor.submit(
                     self.execute_item,
+                    index=i,
                     item=item,
                     params=params,
                     result=result,
                     event=event,
                 )
-                for item in foreach
+                for i, item in enumerate(foreach, start=0)
             ]
             context: DictData = {}
             status: Status = SUCCESS
 
             done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
-            if len(done) != len(futures):
+            if len(list(done)) != len(futures):
                 result.trace.warning(
                     "[STAGE]: Set event for stop pending for-each stage."
                 )
@@ -1741,20 +1784,24 @@ class ForEachStage(BaseStage):
                     future.cancel()
                 time.sleep(0.075)
 
-                nd: str = f", item not run: {not_done}" if not_done else ""
+                nd: str = (
+                    (
+                        f", {len(not_done)} item"
+                        f"{'s' if len(not_done) > 1 else ''} not run!!!"
+                    )
+                    if not_done
+                    else ""
+                )
                 result.trace.debug(
                     f"[STAGE]: ... Foreach-Stage set failed event{nd}"
                 )
-                done: list[Future] = as_completed(futures)
+                done: Iterator[Future] = as_completed(futures)
 
             for future in done:
                 try:
                     future.result()
                 except StageException as e:
                     status = FAILED
-                    result.trace.error(
-                        f"[STAGE]: Error Handler:||{e.__class__.__name__}:||{e}"
-                    )
                     if "errors" in context:
                         context["errors"][e.refs] = e.to_dict()
                     else:
@@ -1764,7 +1811,7 @@ class ForEachStage(BaseStage):
         return result.catch(status=status, context=context)
 
 
-class UntilStage(BaseStage):
+class UntilStage(BaseNestedStage):
     """Until stage executor that will run stages in each loop until it valid
     with stop loop condition.
 
@@ -1815,7 +1862,7 @@ class UntilStage(BaseStage):
         loop: int,
         params: DictData,
         result: Result,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> tuple[Result, T]:
         """Execute all stage with specific loop and item.
 
@@ -1928,8 +1975,8 @@ class UntilStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute until loop with checking until condition.
 
@@ -2015,7 +2062,7 @@ class Match(BaseModel):
     )
 
 
-class CaseStage(BaseStage):
+class CaseStage(BaseNestedStage):
     """Case stage executor that execute all stages if the condition was matched.
 
     Data Validate:
@@ -2066,7 +2113,7 @@ class CaseStage(BaseStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute case.
 
@@ -2151,8 +2198,8 @@ class CaseStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute case-match condition that pass to the case field.
 
@@ -2244,8 +2291,8 @@ class RaiseStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Raise the StageException object with the message field execution.
 
@@ -2259,15 +2306,15 @@ class RaiseStage(BaseAsyncStage):
             extras=self.extras,
         )
         message: str = param2template(self.message, params, extras=self.extras)
-        result.trace.info(f"[STAGE]: Execute Raise-Stage: {message!r}.")
+        result.trace.info(f"[STAGE]: Execute Raise-Stage: ( {message} )")
         raise StageException(message)
 
     async def axecute(
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -2286,7 +2333,7 @@ class RaiseStage(BaseAsyncStage):
             extras=self.extras,
         )
         message: str = param2template(self.message, params, extras=self.extras)
-        await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: {message!r}.")
+        await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
         raise StageException(message)
 
 
@@ -2320,7 +2367,7 @@ class DockerStage(BaseStage):  # pragma: no cov
     env: DictData = Field(
         default_factory=dict,
         description=(
-            "An environment variable that want pass to Docker container.",
+            "An environment variable that want pass to Docker container."
         ),
     )
     volume: DictData = Field(
@@ -2338,7 +2385,7 @@ class DockerStage(BaseStage):  # pragma: no cov
         self,
         params: DictData,
         result: Result,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute Docker container task.
 
@@ -2426,8 +2473,8 @@ class DockerStage(BaseStage):  # pragma: no cov
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Docker image via Python API.
 
@@ -2471,7 +2518,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         py: str,
         values: DictData,
         deps: list[str],
-        run_id: str | None = None,
+        run_id: Optional[str] = None,
     ) -> Iterator[str]:
         """Create the .py file with an input Python string statement.
 
@@ -2479,7 +2526,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         :param values: A variable that want to set before running this
         :param deps: An additional Python dependencies that want install before
             run this python stage.
-        :param run_id: (str | None) A running ID of this stage execution.
+        :param run_id: (Optional[str]) A running ID of this stage execution.
         """
         run_id: str = run_id or uuid.uuid4()
         f_name: str = f"{run_id}.py"
@@ -2518,8 +2565,8 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Python statement via Python virtual environment.