ddeutil-workflow 0.0.75__tar.gz → 0.0.77__tar.gz

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.75
+Version: 0.0.77
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/pyproject.toml

@@ -64,6 +64,7 @@ where = ["src"]
 [tool.shelf.version]
 version = "./src/ddeutil/workflow/__about__.py"
 changelog = "CHANGELOG.md"
+files = ["json-schema.json"]
 commit_msg_format = "- {subject}"
 
 [tool.shelf.git]

ddeutil_workflow-0.0.77/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.77"

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/__cron.py

@@ -843,7 +843,11 @@ class CronRunner:
 
     @property
     def next(self) -> datetime:
-        """Returns the next time of the schedule."""
+        """Returns the next time of the schedule.
+
+        Returns:
+            datetime: A next datetime from the current with shifting step.
+        """
         self.date = (
             self.date
             if self.reset_flag
@@ -860,7 +864,11 @@ class CronRunner:
     def find_date(self, reverse: bool = False) -> datetime:
         """Returns the time the schedule would run by `next` or `prev` methods.
 
-        :param reverse: A reverse flag.
+        Args:
+            reverse: A reverse flag.
+
+        Returns:
+            datetime: A next datetime from shifting step.
         """
         # NOTE: Set reset flag to false if start any action.
         self.reset_flag: bool = False
@@ -870,7 +878,8 @@ class CronRunner:
             max(self.shift_limit, 100) if self.is_year else self.shift_limit
         ):
 
-            # NOTE: Shift the date
+            # NOTE: Shift the date from year to minute.
+            mode: DatetimeMode  # noqa: F842
             if all(
                 not self.__shift_date(mode, reverse)
                 for mode in ("year", "month", "day", "hour", "minute")

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/__init__.py

@@ -113,6 +113,7 @@ from .result import (
     WAIT,
     Result,
     Status,
+    get_status_from_error,
 )
 from .reusables import *
 from .stages import (
@@ -131,7 +132,7 @@ from .stages import (
     VirtualPyStage,
 )
 from .traces import (
-    ConsoleTrace,
+    BaseTrace,
     FileTrace,
     Trace,
     TraceData,

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/audits.py

@@ -79,7 +79,10 @@ class BaseAudit(BaseModel, ABC):
         default=None, description="A parent running ID."
     )
     run_id: str = Field(description="A running ID")
-    execution_time: float = Field(default=0, description="An execution time.")
+    runs_metadata: DictData = Field(
+        default_factory=dict,
+        description="A runs metadata that will use to tracking this audit log.",
+    )
 
     @model_validator(mode="after")
     def __model_action(self) -> Self:
@@ -296,20 +299,22 @@ class FileAudit(BaseAudit):
 
 
 class SQLiteAudit(BaseAudit):  # pragma: no cov
-    """SQLite Audit Pydantic Model."""
+    """SQLite Audit model."""
 
     table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
         str
     ] = """
-        workflow        str,
-        release         int,
-        type            str,
-        context         json,
-        parent_run_id   int,
-        run_id          int,
-        update          datetime
-        primary key ( run_id )
+        workflow          str
+        , release         int
+        , type            str
+        , context         JSON
+        , parent_run_id   int
+        , run_id          int
+        , metadata        JSON
+        , created_at      datetime
+        , updated_at      datetime
+        primary key ( workflow, release )
         """
 
     @classmethod

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/cli.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 import json
 from pathlib import Path
 from platform import python_version
+from textwrap import dedent
 from typing import Annotated, Any, Literal, Optional, Union
 
 import typer
@@ -15,15 +16,13 @@ from pydantic import Field, TypeAdapter
 
 from .__about__ import __version__
 from .__types import DictData
+from .conf import config
 from .errors import JobError
 from .job import Job
 from .params import Param
-from .result import Result
 from .workflow import Workflow
 
-app = typer.Typer(
-    pretty_exceptions_enable=True,
-)
+app = typer.Typer(pretty_exceptions_enable=True)
 
 
 @app.callback()
@@ -41,12 +40,70 @@ def version() -> None:
     typer.echo(f"python-version=={python_version()}")
 
 
+@app.command()
+def init() -> None:
+    """Initialize a Workflow structure on the current context."""
+    config.conf_path.mkdir(exist_ok=True)
+    (config.conf_path / ".confignore").touch()
+
+    conf_example_path: Path = config.conf_path / "examples"
+    conf_example_path.mkdir(exist_ok=True)
+
+    example_template: Path = conf_example_path / "wf_examples.yml"
+    example_template.write_text(
+        dedent(
+            """
+        # Example workflow template.
+        wf-example:
+          type: Workflow
+          desc: |
+            An example workflow template.
+          params:
+            name:
+                type: str
+                default: "World"
+          jobs:
+            first-job:
+              stages:
+                - name: "Call tasks"
+                  uses: tasks/say-hello-func@example
+                  with:
+                    name: ${{ params.name }}
+        """
+        ).lstrip("\n")
+    )
+
+    if "." in config.registry_caller:
+        task_path = Path("./tasks")
+        task_path.mkdir(exist_ok=True)
+
+        dummy_tasks_path = task_path / "example.py"
+        dummy_tasks_path.write_text(
+            dedent(
+                """
+            from ddeutil.workflow import Result, tag
+
+            @tag(name="example", alias="say-hello-func")
+            def hello_world_task(name: str, rs: Result) -> dict[str, str]:
+                \"\"\"Logging hello task function\"\"\"
+                rs.trace.info(f"Hello, {name}")
+                return {"name": name}
+            """
+            ).lstrip("\n")
+        )
+
+        init_path = task_path / "__init__.py"
+        init_path.write_text("from .example import hello_world_task\n")
+    typer.echo(
+        "Starter command: `workflow-cli workflows execute --name=wf-example`"
+    )
+
+
 @app.command(name="job")
 def execute_job(
     params: Annotated[str, typer.Option(help="A job execute parameters")],
     job: Annotated[str, typer.Option(help="A job model")],
-    parent_run_id: Annotated[str, typer.Option(help="A parent running ID")],
-    run_id: Annotated[Optional[str], typer.Option(help="A running ID")] = None,
+    run_id: Annotated[str, typer.Option(help="A running ID")],
 ) -> None:
     """Job execution on the local.
 
@@ -62,26 +119,19 @@ def execute_job(
         job_dict: dict[str, Any] = json.loads(job)
         _job: Job = Job.model_validate(obj=job_dict)
     except json.JSONDecodeError as e:
-        raise ValueError(f"Params does not support format: {params!r}.") from e
+        raise ValueError(f"Jobs does not support format: {job!r}.") from e
 
     typer.echo(f"Job params: {params_dict}")
-    rs: Result = Result(
-        run_id=run_id,
-        parent_run_id=parent_run_id,
-    )
-
     context: DictData = {}
     try:
         _job.set_outputs(
-            _job.execute(
-                params=params_dict,
-                run_id=rs.run_id,
-                parent_run_id=rs.parent_run_id,
-            ).context,
+            _job.execute(params=params_dict, run_id=run_id).context,
             to=context,
         )
+        typer.echo("[JOB]: Context result:")
+        typer.echo(json.dumps(context, default=str, indent=0))
     except JobError as err:
-        rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
+        typer.echo(f"[JOB]: {err.__class__.__name__}: {err}")
 
 
 @app.command()
@@ -136,8 +186,24 @@ def workflow_callback():
 
 
 @workflow_app.command(name="execute")
-def workflow_execute():
-    """"""
+def workflow_execute(
+    name: Annotated[
+        str,
+        typer.Option(help="A name of workflow template."),
+    ],
+    params: Annotated[
+        str,
+        typer.Option(help="A workflow execute parameters"),
+    ] = "{}",
+):
+    """Execute workflow by passing a workflow template name."""
+    try:
+        params_dict: dict[str, Any] = json.loads(params)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
+    typer.echo(f"Start execute workflow template: {name}")
+    typer.echo(f"... with params: {params_dict}")
 
 
 WORKFLOW_TYPE = Literal["Workflow"]
@@ -167,7 +233,7 @@ def workflow_json_schema(
     template_schema: dict[str, str] = {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "title": "Workflow Configuration Schema",
-        "version": "1.0.0",
+        "version": __version__,
     }
     with open(output, mode="w", encoding="utf-8") as f:
         json.dump(template_schema | json_schema, f, indent=2)

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/conf.py

@@ -322,25 +322,30 @@ class YamlParser:
         excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
         ignore_filename: Optional[str] = None,
+        tags: Optional[list[str]] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
-        :param obj: (object | str) An object that want to validate matching
-            before return.
-        :param path: (Path) A config path object.
-        :param paths: (list[Path]) A list of config path object.
-        :param excluded: An included list of data key that want to filter from
-            data.
-        :param extras: (DictData) An extra parameter that use to override core
-            config values.
-        :param ignore_filename: (str) An ignore filename. Default is
+        Args:
+            obj: (object | str) An object that want to validate matching
+                before return.
+            path: (Path) A config path object.
+            paths: (list[Path]) A list of config path object.
+            excluded: An included list of data key that want to filter from
+                data.
+            extras: (DictData) An extra parameter that use to override core
+                config values.
+            ignore_filename: (str) An ignore filename. Default is
             ``.confignore`` filename.
+            tags: (list[str])
+                A list of tag that want to filter.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
         excluded: list[str] = excluded or []
+        tags: list[str] = tags or []
         path: Path = dynamic("conf_path", f=path, extras=extras)
         paths: Optional[list[Path]] = paths or (extras or {}).get("conf_paths")
         if not paths:
@@ -366,6 +371,14 @@ class YamlParser:
                     if key in excluded:
                         continue
 
+                    if (
+                        tags
+                        and (ts := data[key].get("tags"))
+                        and isinstance(ts, list)
+                        and all(t not in tags for t in ts)
+                    ):  # pragma: no cov
+                        continue
+
                     if (t := data.get("type")) and t == obj_type:
                         marking: tuple[float, DictData] = (
                             file.lstat().st_mtime,
@@ -469,7 +482,10 @@ def pass_env(value: T) -> T:  # pragma: no cov
     if isinstance(value, dict):
         return {k: pass_env(value[k]) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([pass_env(i) for i in value])
+        try:
+            return type(value)(pass_env(i) for i in value)
+        except TypeError:
+            return value
     if not isinstance(value, str):
         return value
 

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/errors.py

@@ -136,16 +136,14 @@ class BaseError(Exception):
             ErrorData or dict: Exception data, optionally mapped by reference ID
 
         Example:
-            ```python
-            error = BaseError("Something failed", refs="stage-1")
-
-            # Simple format
-            data = error.to_dict()
-            # Returns: {"name": "BaseError", "message": "Something failed"}
-
-            # With reference mapping
-            ref_data = error.to_dict(with_refs=True)
-            # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
+            >>> error = BaseError("Something failed", refs="stage-1")
+            >>> # Simple format
+            >>> error.to_dict()
+            >>> # Returns: {"name": "BaseError", "message": "Something failed"}
+
+            >>> # With reference mapping
+            >>> error.to_dict(with_refs=True)
+            >>> # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
             ```
         """
         data: ErrorData = to_dict(self)

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/job.py

@@ -656,6 +656,7 @@ class Job(BaseModel):
         to: DictData,
         *,
         job_id: StrOrNone = None,
+        **kwargs,
     ) -> DictData:
         """Set an outputs from execution result context to the received context
         with a `to` input parameter. The result context from job strategy
@@ -693,12 +694,15 @@ class Job(BaseModel):
         :raise JobError: If the job's ID does not set and the setting
             default job ID flag does not set.
 
-        :param output: (DictData) A result data context that want to extract
-            and transfer to the `strategies` key in receive context.
-        :param to: (DictData) A received context data.
-        :param job_id: (StrOrNone) A job ID if the `id` field does not set.
+        Args:
+            output: (DictData) A result data context that want to extract
+                and transfer to the `strategies` key in receive context.
+            to: (DictData) A received context data.
+            job_id: (StrOrNone) A job ID if the `id` field does not set.
+            kwargs: Any values that want to add to the target context.
 
-        :rtype: DictData
+        Returns:
+            DictData: Return updated the target context with a result context.
         """
         if "jobs" not in to:
             to["jobs"] = {}
@@ -716,8 +720,9 @@ class Job(BaseModel):
         status: dict[str, Status] = (
             {"status": output.pop("status")} if "status" in output else {}
         )
+        kwargs: DictData = kwargs or {}
         if self.strategy.is_set():
-            to["jobs"][_id] = {"strategies": output} | errors | status
+            to["jobs"][_id] = {"strategies": output} | errors | status | kwargs
         elif len(k := output.keys()) > 1:  # pragma: no cov
             raise JobError(
                 "Strategy output from execution return more than one ID while "
@@ -726,7 +731,7 @@ class Job(BaseModel):
         else:
             _output: DictData = {} if len(k) == 0 else output[list(k)[0]]
             _output.pop("matrix", {})
-            to["jobs"][_id] = _output | errors | status
+            to["jobs"][_id] = _output | errors | status | kwargs
         return to
 
     def get_outputs(
@@ -800,8 +805,7 @@ class Job(BaseModel):
             return docker_execution(
                 self,
                 params,
-                run_id=run_id,
-                parent_run_id=parent_run_id,
+                run_id=parent_run_id,
                 event=event,
             ).make_info({"execution_time": time.monotonic() - ts})
 
@@ -1294,7 +1298,6 @@ def docker_execution(
     params: DictData,
     *,
     run_id: StrOrNone = None,
-    parent_run_id: StrOrNone = None,
     event: Optional[Event] = None,
 ):  # pragma: no cov
     """Docker job execution.

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/result.py

@@ -16,7 +16,6 @@ Classes:
 Functions:
     validate_statuses: Determine final status from multiple status values
     get_status_from_error: Convert exception types to appropriate status
-    get_dt_tznow: Get current datetime with timezone configuration
 """
 from __future__ import annotations
 
@@ -89,6 +88,7 @@ class Status(str, Enum):
         return self.name
 
     def is_result(self) -> bool:
+        """Return True if this status is the status for result object."""
         return self in ResultStatuses
 
 
@@ -115,15 +115,13 @@ def validate_statuses(statuses: list[Status]) -> Status:
         Status: Final consolidated status based on workflow logic
 
     Example:
-        ```python
-        # Mixed statuses - FAILED takes priority
-        result = validate_statuses([SUCCESS, FAILED, SUCCESS])
-        # Returns: FAILED
-
-        # All same status
-        result = validate_statuses([SUCCESS, SUCCESS, SUCCESS])
-        # Returns: SUCCESS
-        ```
+        >>> # Mixed statuses - FAILED takes priority
+        >>> validate_statuses([SUCCESS, FAILED, SUCCESS])
+        >>> # Returns: FAILED
+
+        >>> # All same status
+        >>> validate_statuses([SUCCESS, SUCCESS, SUCCESS])
+        >>> # Returns: SUCCESS
     """
     if any(s == CANCEL for s in statuses):
         return CANCEL
@@ -153,6 +151,9 @@ def get_status_from_error(
 ) -> Status:
     """Get the Status from the error object.
 
+    Args:
+        error: An error object.
+
     Returns:
         Status: The status from the specific exception class.
     """
@@ -189,7 +190,7 @@ class Result:
     context: DictData = field(default_factory=default_context)
     info: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
-    parent_run_id: Optional[str] = field(default=None, compare=False)
+    parent_run_id: Optional[str] = field(default=None)
     ts: datetime = field(default_factory=get_dt_now, compare=False)
     trace: Optional[Trace] = field(default=None, compare=False, repr=False)
     extras: DictData = field(default_factory=dict, compare=False, repr=False)

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/reusables.py

@@ -421,12 +421,13 @@ def param2template(
             for k in value
         }
     elif isinstance(value, (list, tuple, set)):
-        return type(value)(
-            [
+        try:
+            return type(value)(
                 param2template(i, params, context, filters, extras=extras)
                 for i in value
-            ]
-        )
+            )
+        except TypeError:
+            return value
     elif not isinstance(value, str):
         return value
     return str2template(

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/stages.py

@@ -295,7 +295,7 @@ class BaseStage(BaseModel, ABC):
         ts: float = time.monotonic()
         parent_run_id: str = run_id
         run_id: str = run_id or gen_id(self.iden, unique=True)
-        context: DictData = {}
+        context: DictData = {"status": WAIT}
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
@@ -329,7 +329,7 @@ class BaseStage(BaseModel, ABC):
                 parent_run_id=parent_run_id,
                 event=event,
             )
-            if result_caught.status == WAIT:
+            if result_caught.status == WAIT:  # pragma: no cov
                 raise StageError(
                     "Status from execution should not return waiting status."
                 )
@@ -413,7 +413,7 @@ class BaseStage(BaseModel, ABC):
         self,
         output: DictData,
         to: DictData,
-        info: Optional[DictData] = None,
+        **kwargs,
     ) -> DictData:
         """Set an outputs from execution result context to the received context
         with a `to` input parameter. The result context from stage execution
@@ -447,12 +447,14 @@ class BaseStage(BaseModel, ABC):
         to the `to` argument. The result context was soft copied before set
         output step.
 
-        :param output: (DictData) A result data context that want to extract
-            and transfer to the `outputs` key in receive context.
-        :param to: (DictData) A received context data.
-        :param info: (DictData)
+        Args:
+            output: (DictData) A result data context that want to extract
+                and transfer to the `outputs` key in receive context.
+            to: (DictData) A received context data.
+            kwargs: Any values that want to add to the target context.
 
-        :rtype: DictData
+        Returns:
+            DictData: Return updated the target context with a result context.
         """
         if "stages" not in to:
             to["stages"] = {}
@@ -470,8 +472,8 @@ class BaseStage(BaseModel, ABC):
         status: dict[str, Status] = (
             {"status": output.pop("status")} if "status" in output else {}
         )
-        info: DictData = {"info": info} if info else {}
-        to["stages"][_id] = {"outputs": output} | errors | status | info
+        kwargs: DictData = kwargs or {}
+        to["stages"][_id] = {"outputs": output} | errors | status | kwargs
         return to
 
     def get_outputs(self, output: DictData) -> DictData:
@@ -654,7 +656,7 @@ class BaseAsyncStage(BaseStage, ABC):
                 parent_run_id=parent_run_id,
                 event=event,
             )
-            if result_caught.status == WAIT:
+            if result_caught.status == WAIT:  # pragma: no cov
                 raise StageError(
                     "Status from execution should not return waiting status."
                 )

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/traces.py

@@ -287,7 +287,7 @@ class TraceData(BaseModel):  # pragma: no cov
         return cls.model_validate(data)
 
 
-class BaseTrace(BaseModel, ABC):  # pragma: no cov
+class BaseEmitTrace(BaseModel, ABC):  # pragma: no cov
     """Base Trace model with abstraction class property."""
 
     model_config = ConfigDict(frozen=True)
@@ -474,7 +474,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         await self.amit(message, mode="exception", is_err=True)
 
 
-class ConsoleTrace(BaseTrace):  # pragma: no cov
+class ConsoleTrace(BaseEmitTrace):  # pragma: no cov
     """Console Trace log model."""
 
     def writer(
@@ -566,7 +566,11 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
         getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
 
-class OutsideTrace(ConsoleTrace, ABC):
+class BaseTrace(ConsoleTrace, ABC):
+    """A Base Trace model that will use for override writing or sending trace
+    log to any service type.
+    """
+
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     url: ParseResult = Field(description="An URL for create pointer.")
@@ -575,9 +579,8 @@ class OutsideTrace(ConsoleTrace, ABC):
         "url", mode="before", json_schema_input_type=Union[ParseResult, str]
     )
     def __parse_url(cls, value: Union[ParseResult, str]) -> ParseResult:
-        if isinstance(value, str):
-            return urlparse(value)
-        return value
+        """Parsing an URL value."""
+        return urlparse(value) if isinstance(value, str) else value
 
     @field_serializer("url")
     def __serialize_url(self, value: ParseResult) -> str:
@@ -621,7 +624,7 @@ class OutsideTrace(ConsoleTrace, ABC):
         )
 
 
-class FileTrace(OutsideTrace):  # pragma: no cov
+class FileTrace(BaseTrace):  # pragma: no cov
     """File Trace dataclass that write file to the local storage."""
 
     @classmethod
@@ -765,7 +768,7 @@ class FileTrace(OutsideTrace):  # pragma: no cov
             await f.write(trace_meta.model_dump_json() + "\n")
 
 
-class SQLiteTrace(OutsideTrace):  # pragma: no cov
+class SQLiteTrace(BaseTrace):  # pragma: no cov
     """SQLite Trace dataclass that write trace log to the SQLite database file."""
 
     table_name: ClassVar[str] = "audits"
@@ -779,7 +782,7 @@ class SQLiteTrace(OutsideTrace):  # pragma: no cov
         , metadata          JSON
         , created_at        datetime
         , updated_at        datetime
-        primary key ( run_id )
+        primary key ( parent_run_id )
         """
 
     @classmethod
@@ -824,7 +827,7 @@ class SQLiteTrace(OutsideTrace):  # pragma: no cov
 Trace = Union[
     FileTrace,
     SQLiteTrace,
-    OutsideTrace,
+    BaseTrace,
 ]
 
 

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/utils.py

@@ -271,7 +271,10 @@ def filter_func(value: T) -> T:
     if isinstance(value, dict):
         return {k: filter_func(value[k]) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([filter_func(i) for i in value])
+        try:
+            return type(value)(filter_func(i) for i in value)
+        except TypeError:
+            return value
 
     if isfunction(value):
         # NOTE: If it wants to improve to get this function, it is able to save
@@ -338,7 +341,10 @@ def dump_all(
     if isinstance(value, dict):
         return {k: dump_all(value[k], by_alias=by_alias) for k in value}
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([dump_all(i, by_alias=by_alias) for i in value])
+        try:
+            return type(value)(dump_all(i, by_alias=by_alias) for i in value)
+        except TypeError:
+            return value
     elif isinstance(value, BaseModel):
         return value.model_dump(by_alias=by_alias)
     return value

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil/workflow/workflow.py

@@ -152,6 +152,10 @@ class Workflow(BaseModel):
         default_factory=dict,
         description="A mapping of job ID and job model that already loaded.",
     )
+    tags: list[str] = Field(
+        default_factory=list,
+        description="A list of tag that use for simple grouping workflow.",
+    )
     created_at: datetime = Field(
         default_factory=get_dt_now,
         description=(
@@ -389,11 +393,12 @@ class Workflow(BaseModel):
         params: DictData,
         *,
         run_id: Optional[str] = None,
+        runs_metadata: Optional[DictData] = None,
         release_type: ReleaseType = NORMAL,
-        audit: type[Audit] = None,
         override_log_name: Optional[str] = None,
         timeout: int = 600,
-        excluded: Optional[list[str]] = None,
+        audit_excluded: Optional[list[str]] = None,
+        audit: type[Audit] = None,
     ) -> Result:
         """Release the workflow which is executes workflow with writing audit
         log tracking. The method is overriding parameter with the release
@@ -409,18 +414,22 @@ class Workflow(BaseModel):
             - Execute this workflow with mapping release data to its parameters.
             - Writing result audit
 
-        :param release: (datetime) A release datetime.
-        :param params: A workflow parameter that pass to execute method.
-        :param release_type:
-        :param run_id: (str) A workflow running ID.
-        :param audit: An audit class that want to save the execution result.
-        :param override_log_name: (str) An override logging name that use
-            instead the workflow name.
-        :param timeout: (int) A workflow execution time out in second unit.
-        :param excluded: (list[str]) A list of key that want to exclude from
-            audit data.
+        Args:
+            release: (datetime) A release datetime.
+            params: A workflow parameter that pass to execute method.
+            release_type:
+            run_id: (str) A workflow running ID.
+            runs_metadata: (DictData)
+            audit: An audit class that want to save the execution result.
+            override_log_name: (str) An override logging name that use
+                instead the workflow name.
+            timeout: (int) A workflow execution time out in second unit.
+            audit_excluded: (list[str]) A list of key that want to exclude
+                from the audit data.
 
-        :rtype: Result
+        Returns:
+            Result: return result object that pass context data from the execute
+                method.
         """
         name: str = override_log_name or self.name
 
@@ -432,7 +441,7 @@ class Workflow(BaseModel):
             run_id: str = gen_id(name, unique=True)
             parent_run_id: str = run_id
 
-        context: DictData = {}
+        context: DictData = {"status": WAIT}
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
@@ -445,6 +454,7 @@ class Workflow(BaseModel):
                     "logical_date": release,
                     "execute_date": get_dt_now(),
                     "run_id": run_id,
+                    "runs_metadata": runs_metadata or {},
                 }
             },
             extras=self.extras,
@@ -465,9 +475,17 @@ class Workflow(BaseModel):
                 context=context,
                 parent_run_id=parent_run_id,
                 run_id=run_id,
-                execution_time=rs.info.get("execution_time", 0),
                 extras=self.extras,
-            ).save(excluded=excluded)
+                runs_metadata=(
+                    (runs_metadata or {})
+                    | rs.info
+                    | {
+                        "timeout": timeout,
+                        "original_name": self.name,
+                        "audit_excluded": audit_excluded,
+                    }
+                ),
+            ).save(excluded=audit_excluded)
         )
         return Result(
             run_id=run_id,
@@ -492,7 +510,6 @@ class Workflow(BaseModel):
     def execute_job(
         self,
         job: Job,
-        params: DictData,
         run_id: str,
         context: DictData,
         *,
@@ -511,7 +528,6 @@ class Workflow(BaseModel):
 
         Args:
             job: (Job) A job model that want to execute.
-            params: (DictData) A parameter data.
             run_id: A running stage ID.
             context: A context data.
             parent_run_id: A parent running ID. (Default is None)
@@ -538,25 +554,24 @@ class Workflow(BaseModel):
             )
 
         trace.info(f"[WORKFLOW]: Execute Job: {job.id!r}")
-        rs: Result = job.execute(
-            params=params,
+        result: Result = job.execute(
+            params=context,
             run_id=parent_run_id,
             event=event,
         )
-        job.set_outputs(rs.context, to=params)
+        job.set_outputs(result.context, to=context)
 
-        if rs.status == FAILED:
+        if result.status == FAILED:
             error_msg: str = f"Job execution, {job.id!r}, was failed."
             return FAILED, catch(
                 context=context,
                 status=FAILED,
                 updated={
                     "errors": WorkflowError(error_msg).to_dict(),
-                    **params,
                 },
             )
 
-        elif rs.status == CANCEL:
+        elif result.status == CANCEL:
             error_msg: str = (
                 f"Job execution, {job.id!r}, was canceled from the event after "
                 f"end job execution."
@@ -566,13 +581,10 @@ class Workflow(BaseModel):
                 status=CANCEL,
                 updated={
                     "errors": WorkflowCancelError(error_msg).to_dict(),
-                    **params,
                 },
             )
 
-        return rs.status, catch(
-            context=context, status=rs.status, updated=params
-        )
+        return result.status, catch(context, status=result.status)
 
     def execute(
         self,
@@ -753,7 +765,6 @@ class Workflow(BaseModel):
                         executor.submit(
                             self.execute_job,
                             job=job,
-                            params=context,
                             run_id=run_id,
                             context=context,
                             parent_run_id=parent_run_id,
@@ -768,7 +779,6 @@ class Workflow(BaseModel):
                         executor.submit(
                             self.execute_job,
                             job=job,
-                            params=context,
                             run_id=run_id,
                             context=context,
                             parent_run_id=parent_run_id,
@@ -898,7 +908,7 @@ class Workflow(BaseModel):
                 extras=self.extras,
             )
 
-        err = context["errors"]
+        err: dict[str, str] = context.get("errors", {})
         trace.info(f"[WORKFLOW]: Previous error: {err}")
 
         event: ThreadEvent = event or ThreadEvent()
@@ -919,9 +929,9 @@ class Workflow(BaseModel):
                 extras=self.extras,
             )
 
-        # NOTE: Prepare the new context for rerun process.
+        # NOTE: Prepare the new context variable for rerun process.
         jobs: DictData = context.get("jobs")
-        new_context: DictData = {
+        context: DictData = {
             "params": context["params"].copy(),
             "jobs": {j: jobs[j] for j in jobs if jobs[j]["status"] == SUCCESS},
         }
@@ -930,19 +940,22 @@ class Workflow(BaseModel):
         job_queue: Queue = Queue()
         for job_id in self.jobs:
 
-            if job_id in new_context["jobs"]:
+            if job_id in context["jobs"]:
                 continue
 
             job_queue.put(job_id)
             total_job += 1
 
         if total_job == 0:
-            trace.warning("[WORKFLOW]: It does not have job to rerun.")
+            trace.warning(
+                "[WORKFLOW]: It does not have job to rerun. it will change "
+                "status to skip."
+            )
             return Result(
                 run_id=run_id,
                 parent_run_id=parent_run_id,
-                status=SUCCESS,
-                context=catch(context=context, status=SUCCESS),
+                status=SKIP,
+                context=catch(context=context, status=SKIP),
                 extras=self.extras,
             )
 
@@ -954,14 +967,14 @@ class Workflow(BaseModel):
             "max_job_exec_timeout", f=timeout, extras=self.extras
         )
 
-        catch(new_context, status=WAIT)
+        catch(context, status=WAIT)
         if event and event.is_set():
             return Result(
                 run_id=run_id,
                 parent_run_id=parent_run_id,
                 status=CANCEL,
                 context=catch(
-                    new_context,
+                    context,
                     status=CANCEL,
                     updated={
                         "errors": WorkflowCancelError(
@@ -983,7 +996,7 @@ class Workflow(BaseModel):
             ):
                 job_id: str = job_queue.get()
                 job: Job = self.job(name=job_id)
-                if (check := job.check_needs(new_context["jobs"])) == WAIT:
+                if (check := job.check_needs(context["jobs"])) == WAIT:
                     job_queue.task_done()
                     job_queue.put(job_id)
                     consecutive_waits += 1
@@ -1003,7 +1016,7 @@ class Workflow(BaseModel):
                         parent_run_id=parent_run_id,
                         status=FAILED,
                         context=catch(
-                            new_context,
+                            context,
                             status=FAILED,
                             updated={
                                 "status": FAILED,
@@ -1019,7 +1032,7 @@ class Workflow(BaseModel):
                     trace.info(
                         f"[JOB]: Skip job: {job_id!r} from trigger rule."
                     )
-                    job.set_outputs(output={"status": SKIP}, to=new_context)
+                    job.set_outputs(output={"status": SKIP}, to=context)
                     job_queue.task_done()
                     skip_count += 1
                     continue
@@ -1029,7 +1042,6 @@ class Workflow(BaseModel):
                         executor.submit(
                             self.execute_job,
                             job=job,
-                            params=new_context,
                             run_id=run_id,
                             context=context,
                             parent_run_id=parent_run_id,
@@ -1044,7 +1056,6 @@ class Workflow(BaseModel):
                         executor.submit(
                             self.execute_job,
                             job=job,
-                            params=new_context,
                             run_id=run_id,
                             context=context,
                             parent_run_id=parent_run_id,
@@ -1095,7 +1106,7 @@ class Workflow(BaseModel):
                     run_id=run_id,
                     parent_run_id=parent_run_id,
                     status=st,
-                    context=catch(new_context, status=st),
+                    context=catch(context, status=st),
                     extras=self.extras,
                 )
 
@@ -1115,7 +1126,7 @@ class Workflow(BaseModel):
             parent_run_id=parent_run_id,
             status=FAILED,
             context=catch(
-                new_context,
+                context,
                 status=FAILED,
                 updated={
                     "errors": WorkflowTimeoutError(

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/src/ddeutil_workflow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.75
+Version: 0.0.77
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/tests/test_conf.py

@@ -61,7 +61,10 @@ def test_load_file(target_path: Path):
                     "type": "Workflow",
                     "desc": "Test multi config path",
                     "env": "${WORKFLOW_LOG_TIMEZONE}",
-                }
+                },
+                "test_load_not_set_type": {
+                    "desc": "Test load not set type.",
+                },
             },
             f,
         )
@@ -92,6 +95,12 @@ def test_load_file(target_path: Path):
     with pytest.raises(TypeError):
         YamlParser("test_load_file", extras={"conf_paths": target_path})
 
+    load = YamlParser(
+        "test_load_not_set_type", extras={"conf_paths": [target_path]}
+    )
+    with pytest.raises(ValueError):
+        _ = load.type
+
 
 def test_load_file_finds(target_path: Path):
     dummy_file: Path = target_path / "01_test_simple_file.yaml"
@@ -227,12 +236,13 @@ def test_parse_url():
     assert url.path == "./logs"
 
     url: ParseResult = urlparse("file:///./logs")
-    print(url)
     assert url.scheme == "file"
     assert url.path == "/./logs"
 
     url: ParseResult = urlparse("sqlite:///home/warehouse/sqlite.db")
-    print(url)
+    assert url.scheme == "sqlite"
+    assert url.path == "/home/warehouse/sqlite.db"
 
     url: ParseResult = urlparse("file:./data.db")
-    print(url)
+    assert url.scheme == "file"
+    assert url.path == "./data.db"

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/tests/test_reusables_template.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime
 from typing import Any
+from urllib.parse import urlparse
 
 import pytest
 from ddeutil.workflow.errors import UtilError
@@ -35,11 +36,14 @@ def test_param2template():
                 "${{ params.src }}-${WORKFLOW_LOG_TIMEZONE:-}"
                 "${WORKFLOW_DUMMY:-}"
             ),
+            "url": urlparse("file:./conf"),
+            "set": {"${{ params.src }}", "${{ params.value }}"},
         },
         params={
             "params": {
                 "src": "foo",
                 "value": -10,
+                "url": urlparse("file:./conf"),
             },
         },
     )
@@ -49,6 +53,8 @@ def test_param2template():
         "int_but_str": "value is 10",
         "list": ["foo", -10],
         "str_env": "foo-Asia/Bangkok-",
+        "url": urlparse("file:./conf"),
+        "set": {"foo", -10},
     } == value
 
     with pytest.raises(UtilError):

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/tests/test_utils.py

@@ -14,6 +14,7 @@ from ddeutil.workflow.utils import (
     get_diff_sec,
     get_dt_now,
     make_exec,
+    obj_name,
     prepare_newline,
     reach_next_minute,
 )
@@ -185,3 +186,10 @@ def test_dump_all():
         {"name": "first", "info": {"field": "foo", "age": 10}},
         {"name": "second", "info": {"field": "foo", "age": 10}},
     ]
+
+
+def test_obj_name():
+    assert obj_name() is None
+    assert obj_name("datetime") == "datetime"
+    assert obj_name(datetime) == "datetime"
+    assert obj_name(datetime(2025, 1, 1, 1)) == "datetime"

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/tests/test_workflow_exec_job.py

@@ -18,7 +18,7 @@ def test_workflow_execute_job():
     )
     workflow: Workflow = Workflow(name="workflow", jobs={"demo-run": job})
     st, ctx = workflow.execute_job(
-        job=workflow.job("demo-run"), params={}, run_id="1234", context={}
+        job=workflow.job("demo-run"), run_id="1234", context={}
     )
     assert st == SUCCESS
     assert ctx == {
@@ -46,7 +46,7 @@ def test_workflow_execute_job_raise_inside():
     )
     workflow: Workflow = Workflow(name="workflow", jobs={"demo-run": job})
     st, ctx = workflow.execute_job(
-        job=workflow.job("demo-run"), params={}, run_id="1234", context={}
+        job=workflow.job("demo-run"), run_id="1234", context={}
     )
     assert st == FAILED
     assert ctx == {

{ddeutil_workflow-0.0.75 → ddeutil_workflow-0.0.77}/tests/test_workflow_release.py

@@ -79,6 +79,7 @@ def test_workflow_release():
     rs: Result = workflow.release(
         release=release,
         params={"asat-dt": datetime(2024, 10, 1)},
+        runs_metadata={"runs_by": "nobody"},
     )
     assert rs.status == SUCCESS
     assert rs.context == {

ddeutil_workflow-0.0.75/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.75"