ddeutil-workflow 0.0.57__tar.gz → 0.0.58__tar.gz

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.57
+Version: 0.0.58
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.58/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.58"

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/conf.py

@@ -218,7 +218,7 @@ class BaseLoad(ABC):  # pragma: no cov
 
 class FileLoad(BaseLoad):
     """Base Load object that use to search config data by given some identity
-    value like name of `Workflow` or `On` templates.
+    value like name of `Workflow` or `Crontab` templates.
 
     :param name: (str) A name of key of config data that read with YAML
         Environment object.

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/event.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Event module that store all event object. Now, it has only `On` and `OnYear`
-model these are schedule with crontab event.
+"""Event module that store all event object. Now, it has only `Crontab` and
+`CrontabYear` model these are schedule with crontab event.
 """
 from __future__ import annotations
 
@@ -63,9 +63,9 @@ def interval2crontab(
     return f"{h} {m} {'1' if interval == 'monthly' else '*'} * {d}"
 
 
-class On(BaseModel):
-    """On model (Warped crontab object by Pydantic model) to keep crontab value
-    and generate CronRunner object from this crontab value.
+class Crontab(BaseModel):
+    """Cron event model (Warped the CronJob object by Pydantic model) to keep
+    crontab value and generate CronRunner object from this crontab value.
 
     Methods:
         - generate: is the main use-case of this schedule object.
@@ -128,7 +128,7 @@ class On(BaseModel):
         extras: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config loader that will use loader
-        object for getting the `On` data.
+        object for getting the `Crontab` data.
 
         :param name: (str) A name of config that will get from loader.
         :param extras: (DictData) An extra parameter that use to override core
@@ -172,7 +172,7 @@ class On(BaseModel):
     def __prepare_values(cls, data: Any) -> Any:
         """Extract tz key from value and change name to timezone key.
 
-        :param data: (DictData) A data that want to pass for create an On
+        :param data: (DictData) A data that want to pass for create an Crontab
             model.
 
         :rtype: DictData
@@ -265,9 +265,9 @@ class On(BaseModel):
         return runner
 
 
-class YearOn(On):
-    """On with enhance Year Pydantic model for limit year matrix that use by
-    some data schedule tools like AWS Glue.
+class CrontabYear(Crontab):
+    """Cron event with enhance Year Pydantic model for limit year matrix that
+    use by some data schedule tools like AWS Glue.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/job.py

@@ -839,7 +839,7 @@ def local_execute(
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
     workers: int = job.strategy.max_parallel
     result.trace.info(
-        f"[JOB]: Execute {ls}: {job.id} with {workers} "
+        f"[JOB]: Execute {ls}: {job.id!r} with {workers} "
         f"worker{'s' if workers > 1 else ''}."
     )
 
@@ -886,7 +886,14 @@ def local_execute(
                     future.cancel()
                 time.sleep(0.075)
 
-            nd: str = f", strategies not run: {not_done}" if not_done else ""
+            nd: str = (
+                (
+                    f", {len(not_done)} strateg"
+                    f"{'ies' if len(not_done) > 1 else 'y'} not run!!!"
+                )
+                if not_done
+                else ""
+            )
             result.trace.debug(f"[JOB]: ... Job was set Fail-Fast{nd}")
             done: list[Future] = as_completed(futures)
 

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/logs.py

@@ -70,18 +70,28 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
     return get_dt_now(tz=config.tz)
 
 
+PREFIX_LOGS: dict[str, dict] = {
+    "CALLER": {"emoji": ""},
+    "STAGE": {"emoji": ""},
+    "JOB": {"emoji": ""},
+    "WORKFLOW": {"emoji": "🏃"},
+    "RELEASE": {"emoji": ""},
+    "POKE": {"emoji": ""},
+}  # pragma: no cov
+
+
 class TraceMeta(BaseModel):  # pragma: no cov
     """Trace Metadata model for making the current metadata of this CPU, Memory
     process, and thread data.
     """
 
-    mode: Literal["stdout", "stderr"]
-    datetime: str
-    process: int
-    thread: int
-    message: str
-    filename: str
-    lineno: int
+    mode: Literal["stdout", "stderr"] = Field(description="A meta mode.")
+    datetime: str = Field(description="A datetime in string format.")
+    process: int = Field(description="A process ID.")
+    thread: int = Field(description="A thread ID.")
+    message: str = Field(description="A message log.")
+    filename: str = Field(description="A filename of this log.")
+    lineno: int = Field(description="A line number of this log.")
 
     @classmethod
     def make(
@@ -142,11 +152,9 @@ class TraceData(BaseModel):  # pragma: no cov
         """
         data: DictStr = {"stdout": "", "stderr": "", "meta": []}
 
-        if (file / "stdout.txt").exists():
-            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
-
-        if (file / "stderr.txt").exists():
-            data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
+        for mode in ("stdout", "stderr"):
+            if (file / f"{mode}.txt").exists():
+                data[mode] = (file / f"{mode}.txt").read_text(encoding="utf-8")
 
         if (file / "metadata.json").exists():
             data["meta"] = [
@@ -288,16 +296,30 @@ class BaseTrace(ABC):  # pragma: no cov
         """
         self.__logging(message, mode="exception", is_err=True)
 
+    async def __alogging(
+        self, message: str, mode: str, *, is_err: bool = False
+    ) -> None:
+        """Write trace log with append mode and logging this message with any
+        logging level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = prepare_newline(self.make_message(message))
+
+        if mode != "debug" or (
+            mode == "debug" and dynamic("debug", extras=self.extras)
+        ):
+            await self.awriter(msg, is_err=is_err)
+
+        getattr(logger, mode)(msg, stacklevel=3)
+
     async def adebug(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
         the DEBUG level.
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        if dynamic("debug", extras=self.extras):
-            await self.awriter(msg)
-        logger.info(msg, stacklevel=2)
+        await self.__alogging(message, mode="debug")
 
     async def ainfo(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -305,9 +327,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg)
-        logger.info(msg, stacklevel=2)
+        await self.__alogging(message, mode="info")
 
     async def awarning(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -315,9 +335,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg)
-        logger.warning(msg, stacklevel=2)
+        await self.__alogging(message, mode="warning")
 
     async def aerror(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -325,9 +343,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg, is_err=True)
-        logger.error(msg, stacklevel=2)
+        await self.__alogging(message, mode="error", is_err=True)
 
     async def aexception(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -335,9 +351,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg, is_err=True)
-        logger.exception(msg, stacklevel=2)
+        await self.__alogging(message, mode="exception", is_err=True)
 
 
 class FileTrace(BaseTrace):  # pragma: no cov
@@ -351,7 +365,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
     ) -> Iterator[TraceData]:  # pragma: no cov
         """Find trace logs.
 
-        :param path: (Path)
+        :param path: (Path) A trace path that want to find.
         :param extras: An extra parameter that want to override core config.
         """
         for file in sorted(
@@ -364,16 +378,16 @@ class FileTrace(BaseTrace):  # pragma: no cov
     def find_trace_with_id(
         cls,
         run_id: str,
-        force_raise: bool = True,
         *,
+        force_raise: bool = True,
         path: Path | None = None,
         extras: Optional[DictData] = None,
     ) -> TraceData:
         """Find trace log with an input specific run ID.
 
         :param run_id: A running ID of trace log.
-        :param force_raise:
-        :param path:
+        :param force_raise: (bool)
+        :param path: (Path)
         :param extras: An extra parameter that want to override core config.
         """
         base_path: Path = path or dynamic("trace_path", extras=extras)

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/params.py

@@ -190,10 +190,13 @@ class IntParam(DefaultParam):
 
 
 class FloatParam(DefaultParam):  # pragma: no cov
+    """Float parameter."""
+
     type: Literal["float"] = "float"
     precision: int = 6
 
     def rounding(self, value: float) -> float:
+        """Rounding float value with the specific precision field."""
         round_str: str = f"{{0:.{self.precision}f}}"
         return float(round_str.format(round(value, self.precision)))
 
@@ -224,6 +227,7 @@ class DecimalParam(DefaultParam):  # pragma: no cov
     precision: int = 6
 
     def rounding(self, value: Decimal) -> Decimal:
+        """Rounding float value with the specific precision field."""
         return value.quantize(Decimal(10) ** -self.precision)
 
     def receive(self, value: float | Decimal | None = None) -> Decimal:

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/scheduler.py

@@ -57,7 +57,7 @@ except ImportError:  # pragma: no cov
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
 from .conf import FileLoad, Loader, dynamic
-from .event import On
+from .event import Crontab
 from .exceptions import ScheduleException, WorkflowException
 from .logs import Audit, get_audit
 from .result import SUCCESS, Result
@@ -103,9 +103,9 @@ class ScheduleWorkflow(BaseModel):
         description="An alias name of workflow that use for schedule model.",
     )
     name: str = Field(description="A workflow name.")
-    on: list[On] = Field(
+    on: list[Crontab] = Field(
         default_factory=list,
-        description="An override the list of On object values.",
+        description="An override the list of Crontab object values.",
     )
     values: DictData = Field(
         default_factory=dict,
@@ -158,15 +158,17 @@ class ScheduleWorkflow(BaseModel):
         return data
 
     @field_validator("on", mode="after")
-    def __on_no_dup__(cls, value: list[On], info: ValidationInfo) -> list[On]:
+    def __on_no_dup__(
+        cls, value: list[Crontab], info: ValidationInfo
+    ) -> list[Crontab]:
         """Validate the on fields should not contain duplicate values and if it
         contains every minute value, it should have only one on value.
 
-        :param value: (list[On]) A list of `On` object.
+        :param value: (list[Crontab]) A list of `Crontab` object.
         :param info: (ValidationInfo) An validation info object for getting an
             extra parameter.
 
-        :rtype: list[On]
+        :rtype: list[Crontab]
         """
         set_ons: set[str] = {str(on.cronjob) for on in value}
         if len(set_ons) != len(value):
@@ -209,7 +211,7 @@ class ScheduleWorkflow(BaseModel):
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
         #   field to the Schedule object.
-        ons: list[On] = self.on or wf.on.copy()
+        ons: list[Crontab] = self.on or wf.on.copy()
         workflow_tasks: list[WorkflowTask] = []
         for on in ons:
 

{ddeutil_workflow-0.0.57 → ddeutil_workflow-0.0.58}/src/ddeutil/workflow/stages.py

@@ -507,13 +507,13 @@ class EmptyStage(BaseAsyncStage):
             run_id=gen_id(self.name + (self.id or ""), unique=True),
             extras=self.extras,
         )
-
-        if not self.echo:
-            message: str = "..."
-        else:
-            message: str = param2template(
+        message: str = (
+            param2template(
                 dedent(self.echo.strip("\n")), params, extras=self.extras
             )
+            if self.echo
+            else "..."
+        )
 
         result.trace.info(
             f"[STAGE]: Execute Empty-Stage: {self.name!r}: ( {message} )"
@@ -546,12 +546,13 @@ class EmptyStage(BaseAsyncStage):
             extras=self.extras,
         )
 
-        if not self.echo:
-            message: str = "..."
-        else:
-            message: str = param2template(
+        message: str = (
+            param2template(
                 dedent(self.echo.strip("\n")), params, extras=self.extras
             )
+            if self.echo
+            else "..."
+        )
 
         result.trace.info(f"[STAGE]: Empty-Stage: {self.name!r}: ( {message} )")
         if self.sleep > 0:
@@ -1303,12 +1304,43 @@ class TriggerStage(BaseStage):
                 else "."
             )
             raise StageException(
-                f"Trigger workflow return failed status{err_msg}"
+                f"Trigger workflow return `FAILED` status{err_msg}"
             )
         return rs
 
 
-class ParallelStage(BaseStage):
+class BaseNestedStage(BaseStage):
+    """Base Nested Stage model. This model is use for checking the child stage
+    is the nested stage or not.
+    """
+
+    @abstractmethod
+    def execute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
+    ) -> Result:
+        """Execute abstraction method that action something by sub-model class.
+        This is important method that make this class is able to be the nested
+        stage.
+
+        :param params: (DictData) A parameter data that want to use in this
+            execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        raise NotImplementedError(
+            "Nested-Stage should implement `execute` method."
+        )
+
+
+class ParallelStage(BaseNestedStage):
     """Parallel stage executor that execute branch stages with multithreading.
     This stage let you set the fix branches for running child stage inside it on
     multithread pool.
@@ -1510,19 +1542,14 @@ class ParallelStage(BaseStage):
                     future.result()
                 except StageException as e:
                     status = FAILED
-                    result.trace.error(
-                        f"[STAGE]: Error Handler:||{e.__class__.__name__}:||{e}"
-                    )
                     if "errors" in context:
                         context["errors"][e.refs] = e.to_dict()
                     else:
                         context["errors"] = e.to_dict(with_refs=True)
-                except CancelledError:
-                    pass
         return result.catch(status=status, context=context)
 
 
-class ForEachStage(BaseStage):
+class ForEachStage(BaseNestedStage):
     """For-Each stage executor that execute all stages with each item in the
     foreach list.
 
@@ -1563,9 +1590,17 @@ class ForEachStage(BaseStage):
             "will be sequential mode if this value equal 1."
         ),
     )
+    use_index_as_key: bool = Field(
+        default=False,
+        description=(
+            "A flag for using the loop index as a key instead item value. "
+            "This flag allow to skip checking duplicate item step."
+        ),
+    )
 
     def execute_item(
         self,
+        index: int,
         item: StrOrInt,
         params: DictData,
         result: Result,
@@ -1575,6 +1610,7 @@ class ForEachStage(BaseStage):
         """Execute all nested stage that set on this stage with specific foreach
         item parameter.
 
+        :param index: (int) An index value of foreach loop.
         :param item: (str | int) An item that want to execution.
         :param params: (DictData) A parameter data.
         :param result: (Result) A Result instance for return context and status.
@@ -1588,8 +1624,9 @@ class ForEachStage(BaseStage):
         :rtype: Result
         """
         result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
+        key: StrOrInt = index if self.use_index_as_key else item
         context: DictData = copy.deepcopy(params)
-        context.update({"item": item})
+        context.update({"item": item, "loop": index})
         output: DictData = {"item": item, "stages": {}}
         for stage in self.stages:
 
@@ -1608,14 +1645,14 @@ class ForEachStage(BaseStage):
                 result.catch(
                     status=CANCEL,
                     foreach={
-                        item: {
+                        key: {
                             "item": item,
                             "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         }
                     },
                 )
-                raise StageException(error_msg, refs=item)
+                raise StageException(error_msg, refs=key)
 
             try:
                 rs: Result = stage.handler_execute(
@@ -1631,14 +1668,14 @@ class ForEachStage(BaseStage):
                 result.catch(
                     status=FAILED,
                     foreach={
-                        item: {
+                        key: {
                             "item": item,
                             "stages": filter_func(output.pop("stages", {})),
                             "errors": e.to_dict(),
                         },
                     },
                 )
-                raise StageException(str(e), refs=item) from e
+                raise StageException(str(e), refs=key) from e
 
             if rs.status == FAILED:
                 error_msg: str = (
@@ -1649,19 +1686,19 @@ class ForEachStage(BaseStage):
                 result.catch(
                     status=FAILED,
                     foreach={
-                        item: {
+                        key: {
                             "item": item,
                             "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         },
                     },
                 )
-                raise StageException(error_msg, refs=item)
+                raise StageException(error_msg, refs=key)
 
         return result.catch(
             status=SUCCESS,
             foreach={
-                item: {
+                key: {
                     "item": item,
                     "stages": filter_func(output.pop("stages", {})),
                 },
@@ -1700,6 +1737,11 @@ class ForEachStage(BaseStage):
         # [VALIDATE]: Type of the foreach should be `list` type.
         if not isinstance(foreach, list):
             raise TypeError(f"Does not support foreach: {foreach!r}")
+        elif len(set(foreach)) != len(foreach) and not self.use_index_as_key:
+            raise ValueError(
+                "Foreach item should not duplicate. If this stage must to pass "
+                "duplicate item, it should set `use_index_as_key: true`."
+            )
 
         result.trace.info(f"[STAGE]: Execute Foreach-Stage: {foreach!r}.")
         result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
@@ -1721,12 +1763,13 @@ class ForEachStage(BaseStage):
             futures: list[Future] = [
                 executor.submit(
                     self.execute_item,
+                    index=i,
                     item=item,
                     params=params,
                     result=result,
                     event=event,
                 )
-                for item in foreach
+                for i, item in enumerate(foreach, start=0)
             ]
             context: DictData = {}
             status: Status = SUCCESS
@@ -1741,7 +1784,14 @@ class ForEachStage(BaseStage):
                     future.cancel()
                 time.sleep(0.075)
 
-                nd: str = f", item not run: {not_done}" if not_done else ""
+                nd: str = (
+                    (
+                        f", {len(not_done)} item"
+                        f"{'s' if len(not_done) > 1 else ''} not run!!!"
+                    )
+                    if not_done
+                    else ""
+                )
                 result.trace.debug(
                     f"[STAGE]: ... Foreach-Stage set failed event{nd}"
                 )
@@ -1752,9 +1802,6 @@ class ForEachStage(BaseStage):
                     future.result()
                 except StageException as e:
                     status = FAILED
-                    result.trace.error(
-                        f"[STAGE]: Error Handler:||{e.__class__.__name__}:||{e}"
-                    )
                     if "errors" in context:
                         context["errors"][e.refs] = e.to_dict()
                     else:
@@ -1764,7 +1811,7 @@ class ForEachStage(BaseStage):
         return result.catch(status=status, context=context)
 
 
-class UntilStage(BaseStage):
+class UntilStage(BaseNestedStage):
     """Until stage executor that will run stages in each loop until it valid
     with stop loop condition.
 
@@ -2015,7 +2062,7 @@ class Match(BaseModel):
     )
 
 
-class CaseStage(BaseStage):
+class CaseStage(BaseNestedStage):
     """Case stage executor that execute all stages if the condition was matched.
 
     Data Validate:
@@ -2259,7 +2306,7 @@ class RaiseStage(BaseAsyncStage):
             extras=self.extras,
         )
         message: str = param2template(self.message, params, extras=self.extras)
-        result.trace.info(f"[STAGE]: Execute Raise-Stage: {message!r}.")
+        result.trace.info(f"[STAGE]: Execute Raise-Stage: ( {message} )")
         raise StageException(message)
 
     async def axecute(
@@ -2286,7 +2333,7 @@ class RaiseStage(BaseAsyncStage):
             extras=self.extras,
         )
         message: str = param2template(self.message, params, extras=self.extras)
-        await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: {message!r}.")
+        await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
         raise StageException(message)