ddeutil-workflow 0.0.58__py3-none-any.whl → 0.0.59__py3-none-any.whl

ddeutil/workflow/result.py

@@ -3,16 +3,16 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
-"""Result module. It is the data context transfer objects that use by all object
-in this package. This module provide Status enum object and Result dataclass.
+"""A Result module. It is the data context transfer objects that use by all
+object in this package. This module provide Status enum object and Result
+dataclass.
 """
 from __future__ import annotations
 
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from typing import Optional
+from typing import Optional, Union
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
@@ -22,7 +22,7 @@ from typing_extensions import Self
 from .__types import DictData
 from .conf import dynamic
 from .exceptions import ResultException
-from .logs import Trace, get_dt_tznow, get_trace
+from .logs import TraceModel, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
 
@@ -31,11 +31,11 @@ class Status(IntEnum):
     Result dataclass object.
     """
 
-    SUCCESS: int = 0
-    FAILED: int = 1
-    WAIT: int = 2
-    SKIP: int = 3
-    CANCEL: int = 4
+    SUCCESS = 0
+    FAILED = 1
+    WAIT = 2
+    SKIP = 3
+    CANCEL = 4
 
     @property
     def emoji(self) -> str:
@@ -75,16 +75,16 @@ class Result:
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
+    trace: Optional[TraceModel] = field(default=None, compare=False, repr=False)
     extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @classmethod
     def construct_with_rs_or_id(
         cls,
-        result: Result | None = None,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        id_logic: str | None = None,
+        result: Optional[Result] = None,
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        id_logic: Optional[str] = None,
         *,
         extras: DictData | None = None,
     ) -> Self:
@@ -121,7 +121,7 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: Trace = get_trace(
+            self.trace: TraceModel = get_trace(
                 self.run_id,
                 parent_run_id=self.parent_run_id,
                 extras=self.extras,
@@ -136,14 +136,14 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
-        self.trace: Trace = get_trace(
+        self.trace: TraceModel = get_trace(
             self.run_id, parent_run_id=running_id, extras=self.extras
         )
         return self
 
     def catch(
         self,
-        status: int | Status,
+        status: Union[int, Status],
         context: DictData | None = None,
         **kwargs,
     ) -> Self:

ddeutil/workflow/reusables.py

@@ -7,6 +7,7 @@
 """Reusables module that keep any templating functions."""
 from __future__ import annotations
 
+import copy
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
@@ -35,7 +36,7 @@ logger = logging.getLogger("ddeutil.workflow")
 logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
-FILTERS: dict[str, callable] = {  # pragma: no cov
+FILTERS: dict[str, Callable] = {  # pragma: no cov
     "abs": abs,
     "str": str,
     "int": int,
@@ -258,9 +259,9 @@ def str2template(
     value: str,
     params: DictData,
     *,
-    filters: dict[str, FilterRegistry] | None = None,
+    filters: Optional[dict[str, FilterRegistry]] = None,
     registers: Optional[list[str]] = None,
-) -> str:
+) -> Optional[str]:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
 
@@ -326,7 +327,7 @@ def str2template(
 def param2template(
     value: T,
     params: DictData,
-    filters: dict[str, FilterRegistry] | None = None,
+    filters: Optional[dict[str, FilterRegistry]] = None,
     *,
     extras: Optional[DictData] = None,
 ) -> T:
@@ -381,7 +382,7 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 
 
 @custom_filter("coalesce")  # pragma: no cov
-def coalesce(value: T | None, default: Any) -> T:
+def coalesce(value: Optional[T], default: Any) -> T:
     """Coalesce with default value if the main value is None."""
     return default if value is None else value
 
@@ -450,11 +451,10 @@ def make_registry(
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    regis_calls: list[str] = dynamic(
-        "registry_caller", f=registries
-    )  # pragma: no cov
+    regis_calls: list[str] = copy.deepcopy(
+        dynamic("registry_caller", f=registries)
+    )
     regis_calls.extend(["ddeutil.vendors"])
-
     for module in regis_calls:
         # NOTE: try to sequential import task functions
         try:

ddeutil/workflow/scheduler.py

@@ -335,9 +335,9 @@ class Schedule(BaseModel):
     def pending(
         self,
         *,
-        stop: datetime | None = None,
+        stop: Optional[datetime] = None,
         audit: type[Audit] | None = None,
-        parent_run_id: str | None = None,
+        parent_run_id: Optional[str] = None,
     ) -> Result:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
@@ -384,7 +384,7 @@ DecoratorCancelJob = Callable[[ReturnResultOrCancel], ReturnResultOrCancel]
 
 def catch_exceptions(
     cancel_on_failure: bool = False,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
 ) -> DecoratorCancelJob:
     """Catch exception error from scheduler job that running with schedule
     package and return CancelJob if this function raise an error.
@@ -440,7 +440,7 @@ def schedule_task(
     threads: ReleaseThreads,
     audit: type[Audit],
     *,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
 ) -> ResultOrCancel:
     """Schedule task function that generate thread of workflow task release
@@ -558,7 +558,7 @@ def schedule_task(
 
 def monitor(
     threads: ReleaseThreads,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
 ) -> None:  # pragma: no cov
     """Monitoring function that running every five minute for track long-running
     thread instance from the schedule_control function that run every minute.
@@ -685,11 +685,11 @@ def scheduler_pending(
 
 def schedule_control(
     schedules: list[str],
-    stop: datetime | None = None,
+    stop: Optional[datetime] = None,
     *,
     extras: DictData | None = None,
     audit: type[Audit] | None = None,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
 ) -> Result:  # pragma: no cov
     """Scheduler control function that run the chuck of schedules every minute
     and this function release monitoring thread for tracking undead thread in
@@ -744,7 +744,7 @@ def schedule_control(
 
 
 def schedule_runner(
-    stop: datetime | None = None,
+    stop: Optional[datetime] = None,
     *,
     max_process: int | None = None,
     extras: DictData | None = None,

ddeutil/workflow/stages.py

@@ -141,8 +141,8 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
@@ -162,12 +162,12 @@ class BaseStage(BaseModel, ABC):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-        event: Event | None = None,
-        raise_error: bool | None = None,
-    ) -> Result | DictData:
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+        raise_error: Optional[bool] = None,
+    ) -> Union[Result, DictData]:
         """Handler stage execution result from the stage `execute` method.
 
             This stage exception handler still use ok-error concept, but it
@@ -376,8 +376,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         raise NotImplementedError(
             "Async Stage should implement `execute` method."
@@ -388,8 +388,8 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -411,11 +411,11 @@ class BaseAsyncStage(BaseStage):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        result: Result | None = None,
-        event: Event | None = None,
-        raise_error: bool | None = None,
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+        raise_error: Optional[bool] = None,
     ) -> Result:
         """Async Handler stage execution result from the stage `execute` method.
 
@@ -487,8 +487,8 @@ class EmptyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout.
@@ -528,8 +528,8 @@ class EmptyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -599,13 +599,13 @@ class BashStage(BaseAsyncStage):
 
     @contextlib.asynccontextmanager
     async def acreate_sh_file(
-        self, bash: str, env: DictStr, run_id: str | None = None
+        self, bash: str, env: DictStr, run_id: Optional[str] = None
     ) -> AsyncIterator[TupleStr]:
         """Async create and write `.sh` file with the `aiofiles` package.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (str | None) A running stage ID that use for writing sh
+        :param run_id: (Optional[str]) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: AsyncIterator[TupleStr]
@@ -635,14 +635,14 @@ class BashStage(BaseAsyncStage):
 
     @contextlib.contextmanager
     def create_sh_file(
-        self, bash: str, env: DictStr, run_id: str | None = None
+        self, bash: str, env: DictStr, run_id: Optional[str] = None
     ) -> Iterator[TupleStr]:
         """Create and write the `.sh` file before giving this file name to
         context. After that, it will auto delete this file automatic.
 
         :param bash: (str) A bash statement.
         :param env: (DictStr) An environment variable that set before run bash.
-        :param run_id: (str | None) A running stage ID that use for writing sh
+        :param run_id: (Optional[str]) A running stage ID that use for writing sh
             file instead generate by UUID4.
 
         :rtype: Iterator[TupleStr]
@@ -673,8 +673,8 @@ class BashStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute bash statement with the Python build-in `subprocess` package.
         It will catch result from the `subprocess.run` returning output like
@@ -730,8 +730,8 @@ class BashStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -858,8 +858,8 @@ class PyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on `exec` build-in function.
@@ -916,8 +916,8 @@ class PyStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -1018,8 +1018,8 @@ class CallStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute this caller function with its argument parameter.
 
@@ -1108,8 +1108,8 @@ class CallStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Bash stage that only logging out to
         stdout.
@@ -1266,8 +1266,8 @@ class TriggerStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
@@ -1298,7 +1298,7 @@ class TriggerStage(BaseStage):
             event=event,
         )
         if rs.status == FAILED:
-            err_msg: str | None = (
+            err_msg: Optional[str] = (
                 f" with:\n{msg}"
                 if (msg := rs.context.get("errors", {}).get("message"))
                 else "."
@@ -1319,8 +1319,8 @@ class BaseNestedStage(BaseStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the nested
@@ -1390,7 +1390,7 @@ class ParallelStage(BaseNestedStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute all stage with specific branch ID.
 
@@ -1487,8 +1487,8 @@ class ParallelStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute parallel each branch via multi-threading pool.
 
@@ -1526,7 +1526,7 @@ class ParallelStage(BaseNestedStage):
             context: DictData = {}
             status: Status = SUCCESS
 
-            futures: list[Future] = (
+            futures: list[Future] = [
                 executor.submit(
                     self.execute_branch,
                     branch=branch,
@@ -1535,7 +1535,7 @@ class ParallelStage(BaseNestedStage):
                     event=event,
                 )
                 for branch in self.parallel
-            )
+            ]
 
             for future in as_completed(futures):
                 try:
@@ -1605,7 +1605,7 @@ class ForEachStage(BaseNestedStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute all nested stage that set on this stage with specific foreach
         item parameter.
@@ -1709,8 +1709,8 @@ class ForEachStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the stages that pass each item form the foreach field.
 
@@ -1775,7 +1775,7 @@ class ForEachStage(BaseNestedStage):
             status: Status = SUCCESS
 
             done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
-            if len(done) != len(futures):
+            if len(list(done)) != len(futures):
                 result.trace.warning(
                     "[STAGE]: Set event for stop pending for-each stage."
                 )
@@ -1795,7 +1795,7 @@ class ForEachStage(BaseNestedStage):
                 result.trace.debug(
                     f"[STAGE]: ... Foreach-Stage set failed event{nd}"
                 )
-                done: list[Future] = as_completed(futures)
+                done: Iterator[Future] = as_completed(futures)
 
             for future in done:
                 try:
@@ -1862,7 +1862,7 @@ class UntilStage(BaseNestedStage):
         loop: int,
         params: DictData,
         result: Result,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> tuple[Result, T]:
         """Execute all stage with specific loop and item.
 
@@ -1975,8 +1975,8 @@ class UntilStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute until loop with checking until condition.
 
@@ -2113,7 +2113,7 @@ class CaseStage(BaseNestedStage):
         params: DictData,
         result: Result,
         *,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute case.
 
@@ -2198,8 +2198,8 @@ class CaseStage(BaseNestedStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute case-match condition that pass to the case field.
 
@@ -2291,8 +2291,8 @@ class RaiseStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Raise the StageException object with the message field execution.
 
@@ -2313,8 +2313,8 @@ class RaiseStage(BaseAsyncStage):
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Async execution method for this Empty stage that only logging out to
         stdout.
@@ -2367,7 +2367,7 @@ class DockerStage(BaseStage):  # pragma: no cov
     env: DictData = Field(
         default_factory=dict,
         description=(
-            "An environment variable that want pass to Docker container.",
+            "An environment variable that want pass to Docker container."
         ),
     )
     volume: DictData = Field(
@@ -2385,7 +2385,7 @@ class DockerStage(BaseStage):  # pragma: no cov
         self,
         params: DictData,
         result: Result,
-        event: Event | None = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute Docker container task.
 
@@ -2473,8 +2473,8 @@ class DockerStage(BaseStage):  # pragma: no cov
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Docker image via Python API.
 
@@ -2518,7 +2518,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         py: str,
         values: DictData,
         deps: list[str],
-        run_id: str | None = None,
+        run_id: Optional[str] = None,
     ) -> Iterator[str]:
         """Create the .py file with an input Python string statement.
 
@@ -2526,7 +2526,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         :param values: A variable that want to set before running this
         :param deps: An additional Python dependencies that want install before
             run this python stage.
-        :param run_id: (str | None) A running ID of this stage execution.
+        :param run_id: (Optional[str]) A running ID of this stage execution.
         """
         run_id: str = run_id or uuid.uuid4()
         f_name: str = f"{run_id}.py"
@@ -2565,8 +2565,8 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         self,
         params: DictData,
         *,
-        result: Result | None = None,
-        event: Event | None = None,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Execute the Python statement via Python virtual environment.
 

ddeutil/workflow/utils.py

@@ -15,7 +15,7 @@ from inspect import isfunction
 from itertools import chain, islice, product
 from pathlib import Path
 from random import randrange
-from typing import Any, Final, TypeVar
+from typing import Any, Final, Optional, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import hash_str
@@ -63,7 +63,7 @@ def clear_tz(dt: datetime) -> datetime:
     return dt.replace(tzinfo=None)
 
 
-def get_dt_now(tz: ZoneInfo | None = None, offset: float = 0.0) -> datetime:
+def get_dt_now(tz: Optional[ZoneInfo] = None, offset: float = 0.0) -> datetime:
     """Return the current datetime object.
 
     :param tz: A ZoneInfo object for replace timezone of return datetime object.
@@ -76,7 +76,7 @@ def get_dt_now(tz: ZoneInfo | None = None, offset: float = 0.0) -> datetime:
 
 
 def get_d_now(
-    tz: ZoneInfo | None = None, offset: float = 0.0
+    tz: Optional[ZoneInfo] = None, offset: float = 0.0
 ) -> date:  # pragma: no cov
     """Return the current date object.
 
@@ -147,7 +147,7 @@ def gen_id(
     *,
     sensitive: bool = True,
     unique: bool = False,
-    simple_mode: bool | None = None,
+    simple_mode: Optional[bool] = None,
     extras: DictData | None = None,
 ) -> str:
     """Generate running ID for able to tracking. This generates process use
@@ -197,7 +197,7 @@ def default_gen_id() -> str:
     return gen_id("manual", unique=True)
 
 
-def make_exec(path: str | Path) -> None:
+def make_exec(path: Union[Path, str]) -> None:
     """Change mode of file to be executable file.
 
     :param path: A file path that want to make executable permission.
@@ -244,7 +244,7 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
     )
 
 
-def batch(iterable: Iterator[Any] | range, n: int) -> Iterator[Any]:
+def batch(iterable: Union[Iterator[Any], range], n: int) -> Iterator[Any]:
     """Batch data into iterators of length n. The last batch may be shorter.
 
     Example: