ddeutil-workflow 0.0.58__py3-none-any.whl → 0.0.60__py3-none-any.whl

ddeutil/workflow/reusables.py

@@ -7,6 +7,7 @@
 """Reusables module that keep any templating functions."""
 from __future__ import annotations
 
+import copy
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
@@ -35,7 +36,7 @@ logger = logging.getLogger("ddeutil.workflow")
 logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
-FILTERS: dict[str, callable] = {  # pragma: no cov
+FILTERS: dict[str, Callable] = {  # pragma: no cov
     "abs": abs,
     "str": str,
     "int": int,
@@ -258,9 +259,9 @@ def str2template(
     value: str,
     params: DictData,
     *,
-    filters: dict[str, FilterRegistry] | None = None,
+    filters: Optional[dict[str, FilterRegistry]] = None,
     registers: Optional[list[str]] = None,
-) -> str:
+) -> Optional[str]:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
 
@@ -326,7 +327,7 @@ def str2template(
 def param2template(
     value: T,
     params: DictData,
-    filters: dict[str, FilterRegistry] | None = None,
+    filters: Optional[dict[str, FilterRegistry]] = None,
     *,
     extras: Optional[DictData] = None,
 ) -> T:
@@ -381,7 +382,7 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 
 
 @custom_filter("coalesce")  # pragma: no cov
-def coalesce(value: T | None, default: Any) -> T:
+def coalesce(value: Optional[T], default: Any) -> T:
     """Coalesce with default value if the main value is None."""
     return default if value is None else value
 
@@ -450,11 +451,10 @@ def make_registry(
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    regis_calls: list[str] = dynamic(
-        "registry_caller", f=registries
-    )  # pragma: no cov
+    regis_calls: list[str] = copy.deepcopy(
+        dynamic("registry_caller", f=registries)
+    )
     regis_calls.extend(["ddeutil.vendors"])
-
     for module in regis_calls:
         # NOTE: try to sequential import task functions
         try:

ddeutil/workflow/scheduler.py

@@ -335,9 +335,9 @@ class Schedule(BaseModel):
     def pending(
         self,
         *,
-        stop: datetime | None = None,
+        stop: Optional[datetime] = None,
         audit: type[Audit] | None = None,
-        parent_run_id: str | None = None,
+        parent_run_id: Optional[str] = None,
     ) -> Result:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
@@ -384,7 +384,7 @@ DecoratorCancelJob = Callable[[ReturnResultOrCancel], ReturnResultOrCancel]
 
 def catch_exceptions(
     cancel_on_failure: bool = False,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
 ) -> DecoratorCancelJob:
     """Catch exception error from scheduler job that running with schedule
     package and return CancelJob if this function raise an error.
@@ -440,7 +440,7 @@ def schedule_task(
     threads: ReleaseThreads,
     audit: type[Audit],
     *,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
 ) -> ResultOrCancel:
     """Schedule task function that generate thread of workflow task release
@@ -558,7 +558,7 @@ def schedule_task(
 
 def monitor(
     threads: ReleaseThreads,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
 ) -> None:  # pragma: no cov
     """Monitoring function that running every five minute for track long-running
     thread instance from the schedule_control function that run every minute.
@@ -685,11 +685,11 @@ def scheduler_pending(
 
 def schedule_control(
     schedules: list[str],
-    stop: datetime | None = None,
+    stop: Optional[datetime] = None,
     *,
     extras: DictData | None = None,
     audit: type[Audit] | None = None,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
 ) -> Result:  # pragma: no cov
     """Scheduler control function that run the chuck of schedules every minute
     and this function release monitoring thread for tracking undead thread in
@@ -744,7 +744,7 @@ def schedule_control(
 
 
 def schedule_runner(
-    stop: datetime | None = None,
+    stop: Optional[datetime] = None,
     *,
     max_process: int | None = None,
     extras: DictData | None = None,

ddeutil/workflow/stages.py

@@ -58,7 +58,7 @@ from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, StrOrInt, TupleStr
+from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
 from .conf import dynamic
 from .exceptions import StageException, to_dict
 from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
@@ -87,7 +87,7 @@ class BaseStage(BaseModel, ABC):
         default_factory=dict,
         description="An extra parameter that override core config values.",
     )
-    id: Optional[str] = Field(
+    id: StrOrNone = Field(
         default=None,
         description=(
             "A stage ID that use to keep execution output or getting by job "
@@ -97,7 +97,7 @@ class BaseStage(BaseModel, ABC):
     name: str = Field(
         description="A stage name that want to logging when start execution.",
     )
-    condition: Optional[str] = Field(
+    condition: StrOrNone = Field(
         default=None,
         description=(
             "A stage condition statement to allow stage executable. This field "
@@ -141,8 +141,8 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
@@ -162,12 +162,12 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-        event: Event | None = None,
-        raise_error: bool | None = None,
-    ) -> Result | DictData:
+        run_id: StrOrNone = None,
+        parent_run_id: StrOrNone = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+        raise_error: Optional[bool] = None,
+    ) -> Union[Result, DictData]:
         """Handler stage execution result from the stage `execute` method.
 
             This stage exception handler still use ok-error concept, but it
@@ -376,8 +376,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         raise NotImplementedError(
             "Async Stage should implement `execute` method."
@@ -388,8 +388,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -411,11 +411,11 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-        event: Event | None = None,
-        raise_error: bool | None = None,
+        run_id: StrOrNone = None,
+        parent_run_id: StrOrNone = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+        raise_error: Optional[bool] = None,
     ) -> Result:
         """Async Handler stage execution result from the stage `execute` method.
 
@@ -469,7 +469,7 @@ class EmptyStage(BaseAsyncStage):
         ... }
     """
 
-    echo: Optional[str] = Field(
+    echo: StrOrNone = Field(
         default=None,
         description="A message that want to show on the stdout.",
     )
@@ -487,8 +487,8 @@ class EmptyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout.
@@ -528,8 +528,8 @@ class EmptyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -598,14 +598,14 @@ class BashStage(BaseAsyncStage):
     )
 
     @contextlib.asynccontextmanager
-    async def acreate_sh_file(
-        self, bash: str, env: DictStr, run_id: str | None = None
+    async def async_create_sh_file(
+        self, bash: str, env: DictStr, run_id: StrOrNone = None
     ) -> AsyncIterator[TupleStr]:
         """Async create and write `.sh` file with the `aiofiles` package.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (str | None) A running stage ID that use for writing sh
+        :param run_id: (StrOrNone) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: AsyncIterator[TupleStr]
@@ -635,14 +635,14 @@ class BashStage(BaseAsyncStage):
 
     @contextlib.contextmanager
     def create_sh_file(
-        self, bash: str, env: DictStr, run_id: str | None = None
+        self, bash: str, env: DictStr, run_id: StrOrNone = None
     ) -> Iterator[TupleStr]:
         """Create and write the `.sh` file before giving this file name to
         context. After that, it will auto delete this file automatic.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (str | None) A running stage ID that use for writing sh
+        :param run_id: (StrOrNone) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: Iterator[TupleStr]
@@ -673,8 +673,8 @@ class BashStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute bash statement with the Python build-in `subprocess` package.
         It will catch result from the `subprocess.run` returning output like
@@ -730,8 +730,8 @@ class BashStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -752,7 +752,7 @@ class BashStage(BaseAsyncStage):
             dedent(self.bash.strip("\n")), params, extras=self.extras
         )
 
-        async with self.acreate_sh_file(
+        async with self.async_create_sh_file(
             bash=bash,
             env=param2template(self.env, params, extras=self.extras),
             run_id=result.run_id,
@@ -858,8 +858,8 @@ class PyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on `exec` build-in function.
@@ -916,8 +916,8 @@ class PyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -1018,8 +1018,8 @@ class CallStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute this caller function with its argument parameter.
 
@@ -1108,8 +1108,8 @@ class CallStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -1266,8 +1266,8 @@ class TriggerStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
@@ -1294,11 +1294,12 @@ class TriggerStage(BaseStage):
             extras=self.extras | {"stage_raise_error": True},
         ).execute(
             params=param2template(self.params, params, extras=self.extras),
-            parent_run_id=result.run_id,
+            run_id=None,
+            parent_run_id=result.parent_run_id,
             event=event,
         )
         if rs.status == FAILED:
-            err_msg: str | None = (
+            err_msg: StrOrNone = (
                 f" with:\n{msg}"
                 if (msg := rs.context.get("errors", {}).get("message"))
                 else "."
@@ -1319,8 +1320,8 @@ class BaseNestedStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the nested
@@ -1390,7 +1391,7 @@ class ParallelStage(BaseNestedStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute all stage with specific branch ID.
 
@@ -1487,8 +1488,8 @@ class ParallelStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute parallel each branch via multi-threading pool.
 
@@ -1526,7 +1527,7 @@ class ParallelStage(BaseNestedStage):
             context: DictData = {}
             status: Status = SUCCESS
 
-            futures: list[Future] = (
+            futures: list[Future] = [
                 executor.submit(
                     self.execute_branch,
                     branch=branch,
@@ -1535,7 +1536,7 @@ class ParallelStage(BaseNestedStage):
                     event=event,
                 )
                 for branch in self.parallel
-            )
+            ]
 
             for future in as_completed(futures):
                 try:
@@ -1605,7 +1606,7 @@ class ForEachStage(BaseNestedStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute all nested stage that set on this stage with specific foreach
         item parameter.
@@ -1709,8 +1710,8 @@ class ForEachStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the stages that pass each item form the foreach field.
 
@@ -1775,7 +1776,7 @@ class ForEachStage(BaseNestedStage):
             status: Status = SUCCESS
 
             done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
-            if len(done) != len(futures):
+            if len(list(done)) != len(futures):
                 result.trace.warning(
                     "[STAGE]: Set event for stop pending for-each stage."
                 )
@@ -1795,7 +1796,7 @@ class ForEachStage(BaseNestedStage):
                 result.trace.debug(
                     f"[STAGE]: ... Foreach-Stage set failed event{nd}"
                 )
-                done: list[Future] = as_completed(futures)
+                done: Iterator[Future] = as_completed(futures)
 
             for future in done:
                 try:
@@ -1826,7 +1827,10 @@ class UntilStage(BaseNestedStage):
         ...     "stages": [
         ...         {
         ...             "name": "Start increase item value.",
-        ...             "run": "item = ${{ item }}\\nitem += 1\\n"
+        ...             "run": (
+        ...                 "item = ${{ item }}\\n"
+        ...                 "item += 1\\n"
+        ...             )
         ...         },
         ...     ],
         ... }
@@ -1862,7 +1866,7 @@ class UntilStage(BaseNestedStage):
         loop: int,
         params: DictData,
         result: Result,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> tuple[Result, T]:
         """Execute all stage with specific loop and item.
 
@@ -1975,8 +1979,8 @@ class UntilStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute until loop with checking until condition.
 
@@ -2113,7 +2117,7 @@ class CaseStage(BaseNestedStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute case.
 
@@ -2198,8 +2202,8 @@ class CaseStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute case-match condition that pass to the case field.
 
@@ -2215,9 +2219,7 @@ class CaseStage(BaseNestedStage):
             extras=self.extras,
         )
 
-        _case: Optional[str] = param2template(
-            self.case, params, extras=self.extras
-        )
+        _case: StrOrNone = param2template(self.case, params, extras=self.extras)
 
         result.trace.info(f"[STAGE]: Execute Case-Stage: {_case!r}.")
         _else: Optional[Match] = None
@@ -2291,8 +2293,8 @@ class RaiseStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Raise the StageException object with the message field execution.
 
@@ -2313,8 +2315,8 @@ class RaiseStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -2367,7 +2369,7 @@ class DockerStage(BaseStage):  # pragma: no cov
     env: DictData = Field(
         default_factory=dict,
         description=(
-            "An environment variable that want pass to Docker container.",
+            "An environment variable that want pass to Docker container."
         ),
     )
     volume: DictData = Field(
@@ -2385,7 +2387,7 @@ class DockerStage(BaseStage):  # pragma: no cov
         self,
         params: DictData,
         result: Result,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute Docker container task.
 
@@ -2396,8 +2398,14 @@ class DockerStage(BaseStage):  # pragma: no cov
 
         :rtype: Result
         """
-        from docker import DockerClient
-        from docker.errors import ContainerError
+        try:
+            from docker import DockerClient
+            from docker.errors import ContainerError
+        except ImportError:
+            raise ImportError(
+                "Docker stage need the docker package, you should install it "
+                "by `pip install docker` first."
+            ) from None
 
         client = DockerClient(
             base_url="unix://var/run/docker.sock", version="auto"
@@ -2459,7 +2467,7 @@ class DockerStage(BaseStage):  # pragma: no cov
                 exit_status,
                 None,
                 f"{self.image}:{self.tag}",
-                out,
+                out.decode("utf-8"),
             )
         output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
         if not output_file.exists():
@@ -2473,8 +2481,8 @@ class DockerStage(BaseStage):  # pragma: no cov
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Docker image via Python API.
 
@@ -2518,7 +2526,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         py: str,
         values: DictData,
         deps: list[str],
-        run_id: str | None = None,
+        run_id: StrOrNone = None,
     ) -> Iterator[str]:
         """Create the .py file with an input Python string statement.
 
@@ -2526,7 +2534,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         :param values: A variable that want to set before running this
         :param deps: An additional Python dependencies that want install before
             run this python stage.
-        :param run_id: (str | None) A running ID of this stage execution.
+        :param run_id: (StrOrNone) A running ID of this stage execution.
         """
         run_id: str = run_id or uuid.uuid4()
         f_name: str = f"{run_id}.py"
@@ -2565,8 +2573,8 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Python statement via Python virtual environment.
 
@@ -2644,5 +2652,8 @@ Stage = Annotated[
         RaiseStage,
         EmptyStage,
     ],
-    Field(union_mode="smart"),
+    Field(
+        union_mode="smart",
+        description="A stage models that already implemented on this package.",
+    ),
 ]  # pragma: no cov