ddeutil-workflow 0.0.76__py3-none-any.whl → 0.0.78__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.76"
+__version__: str = "0.0.78"

ddeutil/workflow/cli.py

@@ -57,7 +57,7 @@ def init() -> None:
         wf-example:
           type: Workflow
           desc: |
-            An example workflow template.
+            An example workflow template that provide the demo of workflow.
           params:
             name:
                 type: str
@@ -65,6 +65,10 @@ def init() -> None:
           jobs:
             first-job:
               stages:
+
+                - name: "Hello Stage"
+                  echo: "Start say hi to the console"
+
                 - name: "Call tasks"
                   uses: tasks/say-hello-func@example
                   with:
@@ -232,7 +236,7 @@ def workflow_json_schema(
     json_schema = TypeAdapter(template).json_schema(by_alias=True)
     template_schema: dict[str, str] = {
         "$schema": "http://json-schema.org/draft-07/schema#",
-        "title": "Workflow Configuration Schema",
+        "title": "Workflow Configuration JSON Schema",
         "version": __version__,
     }
     with open(output, mode="w", encoding="utf-8") as f:

ddeutil/workflow/conf.py

@@ -307,8 +307,6 @@ class YamlParser:
                         all_data.append((file_stat.st_mtime, data))
                     elif (t := data.get("type")) and t == obj_type:
                         all_data.append((file_stat.st_mtime, data))
-                    else:
-                        continue
 
         return {} if not all_data else max(all_data, key=lambda x: x[0])[1]
 
@@ -322,25 +320,30 @@ class YamlParser:
         excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
         ignore_filename: Optional[str] = None,
+        tags: Optional[list[Union[str, int]]] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
-        :param obj: (object | str) An object that want to validate matching
-            before return.
-        :param path: (Path) A config path object.
-        :param paths: (list[Path]) A list of config path object.
-        :param excluded: An included list of data key that want to filter from
-            data.
-        :param extras: (DictData) An extra parameter that use to override core
-            config values.
-        :param ignore_filename: (str) An ignore filename. Default is
+        Args:
+            obj: (object | str) An object that want to validate matching
+                before return.
+            path: (Path) A config path object.
+            paths: (list[Path]) A list of config path object.
+            excluded: An included list of data key that want to filter from
+                data.
+            extras: (DictData) An extra parameter that use to override core
+                config values.
+            ignore_filename: (str) An ignore filename. Default is
             ``.confignore`` filename.
+            tags: (list[str])
+                A list of tag that want to filter.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
         excluded: list[str] = excluded or []
+        tags: list[str] = tags or []
         path: Path = dynamic("conf_path", f=path, extras=extras)
         paths: Optional[list[Path]] = paths or (extras or {}).get("conf_paths")
         if not paths:
@@ -366,7 +369,17 @@ class YamlParser:
                     if key in excluded:
                         continue
 
+                    if (
+                        tags
+                        and isinstance((ts := data.get("tags", [])), list)
+                        and any(t not in ts for t in tags)
+                    ):
+                        continue
+
                     if (t := data.get("type")) and t == obj_type:
+                        file_stat: os.stat_result = file.lstat()
+                        data["created_at"] = file_stat.st_ctime
+                        data["updated_at"] = file_stat.st_mtime
                         marking: tuple[float, DictData] = (
                             file.lstat().st_mtime,
                             data,
@@ -469,7 +482,10 @@ def pass_env(value: T) -> T:  # pragma: no cov
     if isinstance(value, dict):
         return {k: pass_env(value[k]) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([pass_env(i) for i in value])
+        try:
+            return type(value)(pass_env(i) for i in value)
+        except TypeError:
+            return value
     if not isinstance(value, str):
         return value
 

ddeutil/workflow/errors.py

@@ -56,13 +56,13 @@ def to_dict(exception: Exception, **kwargs) -> ErrorData:  # pragma: no cov
         ErrorData: Dictionary containing exception name and message
 
     Example:
-        ```python
-        try:
-            raise ValueError("Something went wrong")
-        except Exception as e:
-            error_data = to_dict(e, context="workflow_execution")
-            # Returns: {"name": "ValueError", "message": "Something went wrong", "context": "workflow_execution"}
-        ```
+        >>> try:
+        >>>     raise ValueError("Something went wrong")
+        >>> except Exception as e:
+        >>>     error_data = to_dict(e, context="workflow_execution")
+        >>>     # Returns: {
+        >>>     #   "name": "ValueError", "message": "Something went wrong", "context": "workflow_execution"
+        >>>     # }
     """
     return {
         "name": exception.__class__.__name__,
@@ -85,14 +85,12 @@ class BaseError(Exception):
         params: Parameter data that was being processed when error occurred
 
     Example:
-        ```python
-        try:
-            # Some workflow operation
-            pass
-        except BaseError as e:
-            error_dict = e.to_dict(with_refs=True)
-            print(f"Error in {e.refs}: {error_dict}")
-        ```
+        >>> try:
+        >>>     # NOTE: Some workflow operation
+        >>>     pass
+        >>> except BaseError as e:
+        >>>     error_dict = e.to_dict(with_refs=True)
+        >>>     print(f\"Error in {e.refs}: {error_dict}\")
     """
 
     def __init__(

ddeutil/workflow/event.py

@@ -39,7 +39,6 @@ from __future__ import annotations
 from dataclasses import fields
 from datetime import datetime
 from typing import Annotated, Any, Literal, Optional, Union
-from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.functional_serializers import field_serializer
@@ -107,7 +106,7 @@ class BaseCrontab(BaseModel):
     )
     tz: TimeZoneName = Field(
         default="UTC",
-        description="A timezone string value.",
+        description="A timezone string value that will pass to ZoneInfo.",
         alias="timezone",
     )
 
@@ -125,38 +124,25 @@ class BaseCrontab(BaseModel):
             data["timezone"] = tz
         return data
 
-    @field_validator("tz")
-    def __validate_tz(cls, value: str) -> str:
-        """Validate timezone value.
-
-        Args:
-            value: Timezone string to validate.
-
-        Returns:
-            Validated timezone string.
-
-        Raises:
-            ValueError: If timezone is invalid.
-        """
-        try:
-            _ = ZoneInfo(value)
-            return value
-        except ZoneInfoNotFoundError as e:
-            raise ValueError(f"Invalid timezone: {value}") from e
-
 
 class CrontabValue(BaseCrontab):
     """Crontab model using interval-based specification.
 
     Attributes:
-        interval: Scheduling interval ('daily', 'weekly', 'monthly').
-        day: Day specification for weekly/monthly schedules.
+        interval: (Interval)
+            A scheduling interval string ('daily', 'weekly', 'monthly').
+        day: (str, default None)
+            Day specification for weekly/monthly schedules.
         time: Time of day in 'HH:MM' format.
     """
 
-    interval: Interval
+    interval: Interval = Field(description="A scheduling interval string.")
     day: Optional[str] = Field(default=None)
-    time: str = Field(default="00:00")
+    time: str = Field(
+        default="00:00",
+        pattern=r"\d{2}:\d{2}",
+        description="A time of day that pass with format 'HH:MM'.",
+    )
 
     @property
     def cronjob(self) -> CronJob:
@@ -182,10 +168,13 @@ class CrontabValue(BaseCrontab):
             TypeError: If start parameter is neither string nor datetime.
         """
         if isinstance(start, str):
-            start: datetime = datetime.fromisoformat(start)
-        elif not isinstance(start, datetime):
-            raise TypeError("start value should be str or datetime type.")
-        return self.cronjob.schedule(date=start, tz=self.tz)
+            return self.cronjob.schedule(
+                date=datetime.fromisoformat(start), tz=self.tz
+            )
+
+        if isinstance(start, datetime):
+            return self.cronjob.schedule(date=start, tz=self.tz)
+        raise TypeError("start value should be str or datetime type.")
 
     def next(self, start: Union[str, datetime]) -> CronRunner:
         """Get next scheduled datetime after given start time.
@@ -222,11 +211,6 @@ class Crontab(BaseCrontab):
             "A Cronjob object that use for validate and generate datetime."
         ),
     )
-    tz: TimeZoneName = Field(
-        default="UTC",
-        description="A timezone string value.",
-        alias="timezone",
-    )
 
     @model_validator(mode="before")
     def __prepare_values(cls, data: Any) -> Any:
@@ -376,7 +360,10 @@ Cron = Annotated[
     ],
     Field(
         union_mode="smart",
-        description="Event model type supporting year-based, standard, and interval-based cron scheduling.",
+        description=(
+            "Event model type supporting year-based, standard, and "
+            "interval-based cron scheduling."
+        ),
     ),
 ]  # pragma: no cov
 

ddeutil/workflow/job.py

@@ -48,10 +48,11 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Any, Literal, Optional, Union
+from typing import Annotated, Any, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -263,24 +264,20 @@ class BaseRunsOn(BaseModel):  # pragma: no cov
     object and override execute method.
     """
 
-    type: RunsOn = Field(description="A runs-on type.")
+    type: RunsOn = LOCAL
     args: DictData = Field(
         default_factory=dict,
-        alias="with",
         description=(
             "An argument that pass to the runs-on execution function. This "
             "args will override by this child-model with specific args model."
         ),
+        alias="with",
     )
 
 
 class OnLocal(BaseRunsOn):  # pragma: no cov
     """Runs-on local."""
 
-    type: Literal[RunsOn.LOCAL] = Field(
-        default=RunsOn.LOCAL, validate_default=True
-    )
-
 
 class SelfHostedArgs(BaseModel):
     """Self-Hosted arguments."""
@@ -292,9 +289,7 @@ class SelfHostedArgs(BaseModel):
 class OnSelfHosted(BaseRunsOn):  # pragma: no cov
     """Runs-on self-hosted."""
 
-    type: Literal[RunsOn.SELF_HOSTED] = Field(
-        default=RunsOn.SELF_HOSTED, validate_default=True
-    )
+    type: RunsOn = SELF_HOSTED
     args: SelfHostedArgs = Field(alias="with")
 
 
@@ -310,9 +305,7 @@ class AzBatchArgs(BaseModel):
 
 class OnAzBatch(BaseRunsOn):  # pragma: no cov
 
-    type: Literal[RunsOn.AZ_BATCH] = Field(
-        default=RunsOn.AZ_BATCH, validate_default=True
-    )
+    type: RunsOn = AZ_BATCH
     args: AzBatchArgs = Field(alias="with")
 
 
@@ -331,23 +324,21 @@ class DockerArgs(BaseModel):
 class OnDocker(BaseRunsOn):  # pragma: no cov
     """Runs-on Docker container."""
 
-    type: Literal[RunsOn.DOCKER] = Field(
-        default=RunsOn.DOCKER, validate_default=True
-    )
-    args: DockerArgs = Field(alias="with", default_factory=DockerArgs)
+    type: RunsOn = DOCKER
+    args: DockerArgs = Field(default_factory=DockerArgs, alias="with")
 
 
 def get_discriminator_runs_on(model: dict[str, Any]) -> RunsOn:
     """Get discriminator of the RunsOn models."""
     t: str = model.get("type")
-    return RunsOn(t) if t else RunsOn.LOCAL
+    return RunsOn(t) if t else LOCAL
 
 
 RunsOnModel = Annotated[
     Union[
-        Annotated[OnSelfHosted, Tag(RunsOn.SELF_HOSTED)],
-        Annotated[OnDocker, Tag(RunsOn.DOCKER)],
-        Annotated[OnLocal, Tag(RunsOn.LOCAL)],
+        Annotated[OnSelfHosted, Tag(SELF_HOSTED)],
+        Annotated[OnDocker, Tag(DOCKER)],
+        Annotated[OnLocal, Tag(LOCAL)],
     ],
     Discriminator(get_discriminator_runs_on),
 ]
@@ -490,6 +481,10 @@ class Job(BaseModel):
 
         return self
 
+    @field_serializer("runs_on")
+    def __serialize_runs_on(self, value: RunsOnModel):
+        return value.model_dump(by_alias=True)
+
     def stage(self, stage_id: str) -> Stage:
         """Return stage instance that exists in this job via passing an input
         stage ID.

ddeutil/workflow/result.py

@@ -20,7 +20,6 @@ Functions:
 from __future__ import annotations
 
 from dataclasses import field
-from datetime import datetime
 from enum import Enum
 from typing import Optional, Union
 
@@ -42,7 +41,7 @@ from . import (
 from .__types import DictData
 from .audits import Trace, get_trace
 from .errors import ResultError
-from .utils import default_gen_id, get_dt_now
+from .utils import default_gen_id
 
 
 class Status(str, Enum):
@@ -105,8 +104,9 @@ def validate_statuses(statuses: list[Status]) -> Status:
     """Determine final status from multiple status values.
 
     Applies workflow logic to determine the overall status based on a collection
-    of individual status values. Follows priority order: CANCEL > FAILED > WAIT >
-    individual status consistency.
+    of individual status values. Follows priority order:
+
+        CANCEL > FAILED > WAIT > individual status consistency.
 
     Args:
         statuses: List of status values to evaluate
@@ -132,7 +132,7 @@ def validate_statuses(statuses: list[Status]) -> Status:
     for status in (SUCCESS, SKIP):
         if all(s == status for s in statuses):
             return status
-    return FAILED if FAILED in statuses else SUCCESS
+    return SUCCESS
 
 
 def get_status_from_error(
@@ -166,10 +166,6 @@ def get_status_from_error(
     return FAILED
 
 
-def default_context() -> DictData:
-    return {"status": WAIT}
-
-
 @dataclass(
     config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True),
 )
@@ -186,14 +182,13 @@ class Result:
     field that keep dict value change its ID when update new value to it.
     """
 
+    extras: DictData = field(default_factory=dict, compare=False, repr=False)
     status: Status = field(default=WAIT)
-    context: DictData = field(default_factory=default_context)
+    context: Optional[DictData] = field(default=None)
     info: DictData = field(default_factory=dict)
-    run_id: Optional[str] = field(default_factory=default_gen_id)
+    run_id: str = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None)
-    ts: datetime = field(default_factory=get_dt_now, compare=False)
     trace: Optional[Trace] = field(default=None, compare=False, repr=False)
-    extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @model_validator(mode="after")
     def __prepare_trace(self) -> Self:
@@ -207,20 +202,18 @@ class Result:
                 parent_run_id=self.parent_run_id,
                 extras=self.extras,
             )
-        return self
-
-    def set_parent_run_id(self, running_id: str) -> Self:
-        """Set a parent running ID.
 
-        :param running_id: (str) A running ID that want to update on this model.
+        return self
 
-        :rtype: Self
-        """
-        self.parent_run_id: str = running_id
-        self.trace: Trace = get_trace(
-            self.run_id, parent_run_id=running_id, extras=self.extras
+    @classmethod
+    def from_trace(cls, trace: Trace):
+        """Construct the result model from trace for clean code objective."""
+        return cls(
+            run_id=trace.run_id,
+            parent_run_id=trace.parent_run_id,
+            extras=trace.extras,
+            trace=trace,
         )
-        return self
 
     def catch(
         self,
@@ -237,7 +230,11 @@ class Result:
 
         :rtype: Self
         """
-        self.__dict__["context"].update(context or {})
+        if self.__dict__["context"] is None:
+            self.__dict__["context"] = context
+        else:
+            self.__dict__["context"].update(context or {})
+
         self.__dict__["status"] = (
             Status(status) if isinstance(status, int) else status
         )
@@ -262,13 +259,6 @@ class Result:
         self.__dict__["info"].update(data)
         return self
 
-    def alive_time(self) -> float:  # pragma: no cov
-        """Return total seconds that this object use since it was created.
-
-        :rtype: float
-        """
-        return (get_dt_now() - self.ts).total_seconds()
-
 
 def catch(
     context: DictData,
@@ -276,7 +266,13 @@ def catch(
     updated: DictData | None = None,
     **kwargs,
 ) -> DictData:
-    """Catch updated context to the current context."""
+    """Catch updated context to the current context.
+
+    Args:
+        context: A context data that want to be the current context.
+        status: A status enum object.
+        updated: A updated data that will update to the current context.
+    """
     context.update(updated or {})
     context["status"] = Status(status) if isinstance(status, int) else status
 
@@ -289,7 +285,7 @@ def catch(
             context[k].update(kwargs[k])
         # NOTE: Exclude the `info` key for update information data.
         elif k == "info":
-            context["info"].update(kwargs["info"])
+            context.update({"info": kwargs["info"]})
         else:
             raise ResultError(f"The key {k!r} does not exists on context data.")
     return context

ddeutil/workflow/reusables.py

@@ -421,12 +421,13 @@ def param2template(
             for k in value
         }
     elif isinstance(value, (list, tuple, set)):
-        return type(value)(
-            [
+        try:
+            return type(value)(
                 param2template(i, params, context, filters, extras=extras)
                 for i in value
-            ]
-        )
+            )
+        except TypeError:
+            return value
     elif not isinstance(value, str):
         return value
     return str2template(
@@ -598,7 +599,7 @@ def make_registry(
             if not (
                 hasattr(func, "tag")
                 and hasattr(func, "name")
-                and str(getattr(func, "mark", "NOT SET")) == "tag"
+                and str(getattr(func, "mark", "NOTSET")) == "tag"
             ):  # pragma: no cov
                 continue
 
@@ -616,6 +617,7 @@ def make_registry(
                     f"{module}.{submodule}, you should change this tag name or "
                     f"change it func name."
                 )
+
             rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
 
     return rs

ddeutil/workflow/stages.py

@@ -77,7 +77,15 @@ from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Any, Optional, TypeVar, Union, get_type_hints
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Optional,
+    TypeVar,
+    Union,
+    get_type_hints,
+)
 
 from ddeutil.core import str2list
 from pydantic import BaseModel, Field, ValidationError
@@ -177,7 +185,7 @@ class BaseStage(BaseModel, ABC):
             "A stage description that use to logging when start execution."
         ),
     )
-    condition: StrOrNone = Field(
+    condition: Optional[Union[str, bool]] = Field(
         default=None,
         description=(
             "A stage condition statement to allow stage executable. This field "
@@ -329,7 +337,7 @@ class BaseStage(BaseModel, ABC):
                 parent_run_id=parent_run_id,
                 event=event,
             )
-            if result_caught.status == WAIT:
+            if result_caught.status == WAIT:  # pragma: no cov
                 raise StageError(
                     "Status from execution should not return waiting status."
                 )
@@ -513,6 +521,9 @@ class BaseStage(BaseModel, ABC):
         if not self.condition:
             return False
 
+        if isinstance(self.condition, bool):
+            return self.condition
+
         try:
             # WARNING: The eval build-in function is very dangerous. So, it
             #   should use the `re` module to validate eval-string before
@@ -656,7 +667,7 @@ class BaseAsyncStage(BaseStage, ABC):
                 parent_run_id=parent_run_id,
                 event=event,
             )
-            if result_caught.status == WAIT:
+            if result_caught.status == WAIT:  # pragma: no cov
                 raise StageError(
                     "Status from execution should not return waiting status."
                 )
@@ -1580,15 +1591,23 @@ class CallStage(BaseRetryStage):
 
         :rtype: Any
         """
-        if isinstance(value, dict):
-            if any(k in value for k in ("result", "extras")):
-                raise ValueError(
-                    "The argument on workflow template for the caller stage "
-                    "should not pass `result` and `extras`. They are special "
-                    "arguments."
-                )
+        if isinstance(value, dict) and any(
+            k in value for k in ("result", "extras")
+        ):
+            raise ValueError(
+                "The argument on workflow template for the caller stage "
+                "should not pass `result` and `extras`. They are special "
+                "arguments."
+            )
         return value
 
+    def get_caller(self, params: DictData) -> Callable[[], TagFunc]:
+        """Get the lazy TagFuc object from registry."""
+        return extract_call(
+            param2template(self.uses, params, extras=self.extras),
+            registries=self.extras.get("registry_caller"),
+        )
+
     def process(
         self,
         params: DictData,
@@ -1615,11 +1634,7 @@ class CallStage(BaseRetryStage):
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
-        call_func: TagFunc = extract_call(
-            param2template(self.uses, params, extras=self.extras),
-            registries=self.extras.get("registry_caller"),
-        )()
-
+        call_func: TagFunc = self.get_caller(params=params)()
         trace.info(f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'")
 
         # VALIDATE: check input task caller parameters that exists before
@@ -1677,7 +1692,7 @@ class CallStage(BaseRetryStage):
             )
 
         args: DictData = self.validate_model_args(
-            call_func, args, run_id, parent_run_id
+            call_func, args, run_id, parent_run_id, extras=self.extras
         )
         if inspect.iscoroutinefunction(call_func):
             loop = asyncio.get_event_loop()
@@ -1738,11 +1753,7 @@ class CallStage(BaseRetryStage):
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
-        call_func: TagFunc = extract_call(
-            param2template(self.uses, params, extras=self.extras),
-            registries=self.extras.get("registry_caller"),
-        )()
-
+        call_func: TagFunc = self.get_caller(params=params)()
         await trace.ainfo(
             f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
         )
@@ -1795,8 +1806,13 @@ class CallStage(BaseRetryStage):
         if "extras" not in sig.parameters and not has_keyword:
             args.pop("extras")
 
+        if event and event.is_set():
+            raise StageCancelError(
+                "Execution was canceled from the event before start parallel."
+            )
+
         args: DictData = self.validate_model_args(
-            call_func, args, run_id, parent_run_id
+            call_func, args, run_id, parent_run_id, extras=self.extras
         )
         if inspect.iscoroutinefunction(call_func):
             rs: DictOrModel = await call_func(
@@ -1829,12 +1845,13 @@ class CallStage(BaseRetryStage):
             extras=self.extras,
         )
 
+    @staticmethod
     def validate_model_args(
-        self,
         func: TagFunc,
         args: DictData,
         run_id: str,
         parent_run_id: Optional[str] = None,
+        extras: Optional[DictData] = None,
     ) -> DictData:
         """Validate an input arguments before passing to the caller function.
 
@@ -1846,18 +1863,18 @@ class CallStage(BaseRetryStage):
         :rtype: DictData
         """
         try:
-            model_instance: BaseModel = create_model_from_caller(
-                func
-            ).model_validate(args)
-            override: DictData = dict(model_instance)
+            override: DictData = dict(
+                create_model_from_caller(func).model_validate(args)
+            )
             args.update(override)
+
             type_hints: dict[str, Any] = get_type_hints(func)
             for arg in type_hints:
 
                 if arg == "return":
                     continue
 
-                if arg.removeprefix("_") in args:
+                if arg.startswith("_") and arg.removeprefix("_") in args:
                     args[arg] = args.pop(arg.removeprefix("_"))
                     continue
 
@@ -1868,7 +1885,7 @@ class CallStage(BaseRetryStage):
             ) from e
         except TypeError as e:
             trace: Trace = get_trace(
-                run_id, parent_run_id=parent_run_id, extras=self.extras
+                run_id, parent_run_id=parent_run_id, extras=extras
             )
             trace.warning(
                 f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
@@ -2295,7 +2312,13 @@ class ForEachStage(BaseNestedStage):
         ... }
     """
 
-    foreach: Union[list[str], list[int], str] = Field(
+    foreach: Union[
+        list[str],
+        list[int],
+        str,
+        dict[str, Any],
+        dict[int, Any],
+    ] = Field(
         description=(
             "A items for passing to stages via ${{ item }} template parameter."
         ),
@@ -2501,7 +2524,7 @@ class ForEachStage(BaseNestedStage):
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         event: Event = event or Event()
-        foreach: Union[list[str], list[int]] = pass_env(
+        foreach: Union[list[str], list[int], str] = pass_env(
             param2template(self.foreach, params, extras=self.extras)
         )
 
@@ -2516,9 +2539,10 @@ class ForEachStage(BaseNestedStage):
                 ) from e
 
         # [VALIDATE]: Type of the foreach should be `list` type.
-        elif not isinstance(foreach, list):
+        elif isinstance(foreach, dict):
             raise TypeError(
-                f"Does not support foreach: {foreach!r} ({type(foreach)})"
+                f"Does not support dict foreach: {foreach!r} ({type(foreach)}) "
+                f"yet."
             )
         # [Validate]: Value in the foreach item should not be duplicate when the
         #   `use_index_as_key` field did not set.

ddeutil/workflow/traces.py

@@ -32,6 +32,7 @@ Example:
     >>> trace = get_trace("running-id-101", parent_run_id="workflow-001")
     >>> trace.info("Workflow execution started")
     >>> trace.debug("Processing stage 1")
+
 """
 from __future__ import annotations
 
@@ -848,7 +849,7 @@ def get_trace(
     :rtype: Trace
     """
     # NOTE: Allow you to override trace model by the extra parameter.
-    map_trace_models: dict[str, type[Trace]] = extras.get(
+    map_trace_models: dict[str, type[Trace]] = (extras or {}).get(
         "trace_model_mapping", {}
     )
     url: ParseResult

ddeutil/workflow/utils.py

@@ -247,7 +247,7 @@ def default_gen_id() -> str:
 
     :rtype: str
     """
-    return gen_id("manual", unique=True)
+    return gen_id("MOCK", unique=True)
 
 
 def make_exec(path: Union[Path, str]) -> None:
@@ -271,7 +271,10 @@ def filter_func(value: T) -> T:
     if isinstance(value, dict):
         return {k: filter_func(value[k]) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([filter_func(i) for i in value])
+        try:
+            return type(value)(filter_func(i) for i in value)
+        except TypeError:
+            return value
 
     if isfunction(value):
         # NOTE: If it wants to improve to get this function, it is able to save
@@ -338,7 +341,10 @@ def dump_all(
     if isinstance(value, dict):
         return {k: dump_all(value[k], by_alias=by_alias) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([dump_all(i, by_alias=by_alias) for i in value])
+        try:
+            return type(value)(dump_all(i, by_alias=by_alias) for i in value)
+        except TypeError:
+            return value
     elif isinstance(value, BaseModel):
         return value.model_dump(by_alias=by_alias)
     return value
@@ -346,7 +352,7 @@ def dump_all(
 
 def obj_name(obj: Optional[Union[str, object]] = None) -> Optional[str]:
     if not obj:
-        obj_type: Optional[str] = None
+        return None
     elif isinstance(obj, str):
         obj_type: str = obj
     elif isclass(obj):

ddeutil/workflow/workflow.py

@@ -145,13 +145,17 @@ class Workflow(BaseModel):
         description="A parameters that need to use on this workflow.",
     )
     on: Event = Field(
-        default_factory=list,
+        default_factory=Event,
         description="An events for this workflow.",
     )
     jobs: dict[str, Job] = Field(
         default_factory=dict,
         description="A mapping of job ID and job model that already loaded.",
     )
+    tags: list[str] = Field(
+        default_factory=list,
+        description="A list of tag that use for simple grouping workflow.",
+    )
     created_at: datetime = Field(
         default_factory=get_dt_now,
         description=(
@@ -207,11 +211,6 @@ class Workflow(BaseModel):
             ```
         """
         load: YamlParser = YamlParser(name, path=path, extras=extras, obj=cls)
-
-        # NOTE: Validate the config type match with current connection model
-        if load.type != cls.__name__:
-            raise ValueError(f"Type {load.type} does not match with {cls}")
-
         data: DictData = copy.deepcopy(load.data)
         data["name"] = name
 
@@ -285,6 +284,50 @@ class Workflow(BaseModel):
 
         return self
 
+    def detail(self) -> DictData:  # pragma: no cov
+        """Return the detail of this workflow for generate markdown."""
+        return self.model_dump(by_alias=True)
+
+    def md(self, author: Optional[str] = None) -> str:  # pragma: no cov
+        """Generate the markdown template."""
+
+        def align_newline(value: str) -> str:
+            return value.rstrip("\n").replace("\n", "\n                ")
+
+        info: str = (
+            f"| Author: {author or 'nobody'} "
+            f"| created_at: `{self.created_at:%Y-%m-%d %H:%M:%S}` "
+            f"| updated_at: `{self.updated_dt:%Y-%m-%d %H:%M:%S}` |\n"
+            f"| --- | --- | --- |"
+        )
+        jobs: str = ""
+        for job in self.jobs:
+            job_model: Job = self.jobs[job]
+            jobs += f"### {job}\n{job_model.desc or ''}\n"
+            stags: str = ""
+            for stage_model in job_model.stages:
+                stags += (
+                    f"#### {stage_model.name}\n\n"
+                    f"Stage ID: {stage_model.id or ''}\n"
+                    f"Stage Model: {stage_model.__class__.__name__}\n\n"
+                )
+            jobs += f"{stags}\n"
+        return dedent(
+            f"""
+                # Workflow: {self.name}\n
+                {align_newline(info)}\n
+                {align_newline(self.desc)}\n
+                ## Parameters\n
+                | name | type | default | description |
+                | --- | --- | --- | : --- : |
+
+                ## Jobs\n
+                {align_newline(jobs)}
+                """.lstrip(
+                "\n"
+            )
+        )
+
     def job(self, name: str) -> Job:
         """Return the workflow's Job model that getting by an input job's name
         or job's ID. This method will pass an extra parameter from this model
@@ -373,7 +416,7 @@ class Workflow(BaseModel):
             dt = dt.replace(tzinfo=UTC)
 
         release: datetime = replace_sec(dt.astimezone(UTC))
-        if not self.on:
+        if not self.on.schedule:
             return release
 
         for on in self.on.schedule:
@@ -867,7 +910,7 @@ class Workflow(BaseModel):
         event: Optional[ThreadEvent] = None,
         timeout: float = 3600,
         max_job_parallel: int = 2,
-    ) -> Result:
+    ) -> Result:  # pragma: no cov
         """Re-Execute workflow with passing the error context data.
 
         :param context: A context result that get the failed status.
@@ -896,12 +939,9 @@ class Workflow(BaseModel):
                 "[WORKFLOW]: Does not rerun because it already executed with "
                 "success status."
             )
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=SUCCESS,
                 context=catch(context=context, status=SUCCESS),
-                extras=self.extras,
             )
 
         err: dict[str, str] = context.get("errors", {})
@@ -917,12 +957,9 @@ class Workflow(BaseModel):
         )
         if not self.jobs:
             trace.warning(f"[WORKFLOW]: {self.name!r} does not set jobs")
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=SUCCESS,
                 context=catch(context=context, status=SUCCESS),
-                extras=self.extras,
             )
 
         # NOTE: Prepare the new context variable for rerun process.
@@ -947,12 +984,9 @@ class Workflow(BaseModel):
                 "[WORKFLOW]: It does not have job to rerun. it will change "
                 "status to skip."
             )
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=SKIP,
                 context=catch(context=context, status=SKIP),
-                extras=self.extras,
             )
 
         not_timeout_flag: bool = True
@@ -965,9 +999,7 @@ class Workflow(BaseModel):
 
         catch(context, status=WAIT)
         if event and event.is_set():
-            return Result(
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+            return Result.from_trace(trace).catch(
                 status=CANCEL,
                 context=catch(
                     context,
@@ -979,7 +1011,6 @@ class Workflow(BaseModel):
                         ).to_dict(),
                     },
                 ),
-                extras=self.extras,
             )
 
         with ThreadPoolExecutor(max_job_parallel, "wf") as executor:
@@ -1007,9 +1038,7 @@ class Workflow(BaseModel):
                 backoff_sleep = 0.01
 
                 if check == FAILED:  # pragma: no cov
-                    return Result(
-                        run_id=run_id,
-                        parent_run_id=parent_run_id,
+                    return Result.from_trace(trace).catch(
                         status=FAILED,
                         context=catch(
                             context,
@@ -1022,7 +1051,6 @@ class Workflow(BaseModel):
                                 ).to_dict(),
                             },
                         ),
-                        extras=self.extras,
                     )
                 elif check == SKIP:  # pragma: no cov
                     trace.info(
@@ -1098,12 +1126,9 @@ class Workflow(BaseModel):
                     statuses[total + 1 + skip_count + i] = s
 
                 st: Status = validate_statuses(statuses)
-                return Result(
-                    run_id=run_id,
-                    parent_run_id=parent_run_id,
+                return Result.from_trace(trace).catch(
                     status=st,
                     context=catch(context, status=st),
-                    extras=self.extras,
                 )
 
             event.set()
@@ -1117,9 +1142,7 @@ class Workflow(BaseModel):
 
             time.sleep(0.0025)
 
-        return Result(
-            run_id=run_id,
-            parent_run_id=parent_run_id,
+        return Result.from_trace(trace).catch(
             status=FAILED,
             context=catch(
                 context,
@@ -1131,5 +1154,4 @@ class Workflow(BaseModel):
                     ).to_dict(),
                 },
             ),
-            extras=self.extras,
         )

{ddeutil_workflow-0.0.76.dist-info → ddeutil_workflow-0.0.78.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.76
+Version: 0.0.78
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -147,7 +147,6 @@ For comprehensive API documentation, examples, and best practices:
 - **[Full Documentation](https://ddeutils.github.io/ddeutil-workflow/)** - Complete user guide and API reference
 - **[Getting Started](https://ddeutils.github.io/ddeutil-workflow/getting-started/)** - Quick start guide
 - **[API Reference](https://ddeutils.github.io/ddeutil-workflow/api/workflow/)** - Detailed API documentation
-- **[Examples](https://ddeutils.github.io/ddeutil-workflow/examples/)** - Real-world usage examples
 
 ## 🎯 Usage
 

ddeutil_workflow-0.0.78.dist-info/RECORD

@@ -0,0 +1,30 @@
+ddeutil/workflow/__about__.py,sha256=9g_DJubkaHxt4W_0-r1PmUT5x3bCo_72c-f60EDtlq8,28
+ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
+ddeutil/workflow/__init__.py,sha256=_8sP-CTPOfwsFFhmdwQ2Gp7yY7qJemP7TYsIWgd5jc0,3300
+ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
+ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
+ddeutil/workflow/audits.py,sha256=wANG0jEQ7slUSgVZG4JbjlR5PtmF8mHpM9RH-zpYM_g,12679
+ddeutil/workflow/cli.py,sha256=WnUkxqs2hCc5JVuTvWuEGKp8_EcS_wmhVvSXDhj0eEM,7544
+ddeutil/workflow/conf.py,sha256=7yMSBVh2W9KcDOZxQjeabJZizB_3ydCLICo4JI0syWU,16892
+ddeutil/workflow/errors.py,sha256=UpUIqoyqkvzqjuxtUQ9535l1HeAsyh-plEG0PgDVR2w,5541
+ddeutil/workflow/event.py,sha256=6d5_UvnPI8xRLcX_5wptmvWoXUIGs_JZbjq7khz5oYE,13355
+ddeutil/workflow/job.py,sha256=4_IxtoIwl1qVuSaNngelYNkqQl9ZOa-PHdKCnXvXu0M,43943
+ddeutil/workflow/params.py,sha256=Cyz142OcvENIZrM7Efc2xuGPmmFBhROifP5ojoaCezg,13658
+ddeutil/workflow/result.py,sha256=Xi07E3WuMHS1jLcJg7p4DPuaMFGp0yEDaWCJRotOH6g,8921
+ddeutil/workflow/reusables.py,sha256=pbCHsEl2V3jGWDRcGyxDvGN5rP5kaRxZNgv9x6-pINQ,23338
+ddeutil/workflow/stages.py,sha256=ocsk64duS4BHEOLT3Agkx-fbY6iYmygxCaQhwp6YyyM,122482
+ddeutil/workflow/traces.py,sha256=h7oDlb4Q8LJUp0pste2dWJYOqEaN64KsLTNMfeRUqx8,28475
+ddeutil/workflow/utils.py,sha256=vSGdpaFgQ5vUPxWvVfbNC2__tu5q16B9mhx1BRbEuJo,10968
+ddeutil/workflow/workflow.py,sha256=0KaRCTAipPvvmi9SOC7T0V0CAusgebCM6qju0ethGF0,42409
+ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
+ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
+ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
+ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
+ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
+ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
+ddeutil_workflow-0.0.78.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.78.dist-info/METADATA,sha256=2phJ3JRH9o0Sbr22iOo1YeQBr8nyYx6areM2rwNp_Eg,15681
+ddeutil_workflow-0.0.78.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddeutil_workflow-0.0.78.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.78.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.78.dist-info/RECORD,,

ddeutil_workflow-0.0.76.dist-info/RECORD

@@ -1,30 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=ZWCr4NXnXaxhYuhduVXp9CkXr2m6FgZDruUsw8AdJBM,28
-ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
-ddeutil/workflow/__init__.py,sha256=_8sP-CTPOfwsFFhmdwQ2Gp7yY7qJemP7TYsIWgd5jc0,3300
-ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
-ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
-ddeutil/workflow/audits.py,sha256=wANG0jEQ7slUSgVZG4JbjlR5PtmF8mHpM9RH-zpYM_g,12679
-ddeutil/workflow/cli.py,sha256=eAwRZMSEJu-NONc_un0D_1swFlENMjl3C-iXYnyTTPY,7411
-ddeutil/workflow/conf.py,sha256=UCw6v2GFD3tA2LRbp7vLifXniey0P5Ef0U9eBPknrWk,16267
-ddeutil/workflow/errors.py,sha256=tHS6ekxBmZ6sIeLaxHHSaMfhVvlWnndfb2-Aq-bL2So,5509
-ddeutil/workflow/event.py,sha256=siChcBhsu4ejzW1fK0tjHPXQVaSUCSxPYDgDrh6duwo,13676
-ddeutil/workflow/job.py,sha256=_NOPWPs2FuiMvNE-L6c9mpXEChXmgQ8zmD33ZzqVi0A,44146
-ddeutil/workflow/params.py,sha256=Cyz142OcvENIZrM7Efc2xuGPmmFBhROifP5ojoaCezg,13658
-ddeutil/workflow/result.py,sha256=Fz6y6apivLW-94gAxcT42z-mGqWMk6-O3RJ2GGSNUHM,9146
-ddeutil/workflow/reusables.py,sha256=q_OA-oifCGIhW_5j6hTZXZk7FBOmDt0xVrtNnscJfNg,23294
-ddeutil/workflow/stages.py,sha256=Z6quvhJ5WZPnItd4xOoQyR_KWE2Z6LYWa5d49N0R5D8,121936
-ddeutil/workflow/traces.py,sha256=0n6Mytp6oeNjOV8lIsFitzZ6TrtuSNVFkUmodBiE_vA,28466
-ddeutil/workflow/utils.py,sha256=EXhIuWzOJHvlcoAdyvuDUomGtMTIB59HxOLpj2VJ1bI,10857
-ddeutil/workflow/workflow.py,sha256=Yw7xuEIwQ61qhGfElky9AZY_1o_1Gqta4B1x1nwQNJs,41475
-ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
-ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
-ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
-ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
-ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
-ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
-ddeutil_workflow-0.0.76.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.76.dist-info/METADATA,sha256=zq63qmHeFG1DpJJWKyO5IjA6tMbYQnHNWa8E5OESD5w,15781
-ddeutil_workflow-0.0.76.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddeutil_workflow-0.0.76.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
-ddeutil_workflow-0.0.76.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.76.dist-info/RECORD,,