ddeutil-workflow 0.0.44__tar.gz → 0.0.45__tar.gz

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.44
+Version: 0.0.45
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -30,7 +30,6 @@ Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: all
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
 Requires-Dist: httpx; extra == "all"
-Requires-Dist: ujson; extra == "all"
 Requires-Dist: aiofiles; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Provides-Extra: api
@@ -71,9 +70,9 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. The Minimum frequency unit of scheduling is **1 Minute** 🕘
+1. The Minimum frequency unit of built-in scheduling is **1 Minute** 🕘
 2. **Can not** re-run only failed stage and its pending downstream ↩️
-3. All parallel tasks inside workflow engine use **Multi-Threading**
+3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
    (Python 3.13 unlock GIL 🐍🔓)
 
 ---
@@ -266,11 +265,11 @@ it will use default value and do not raise any error to you.
 | **ROOT_PATH**                |   Core    | `.`                                                                                                                             |    No    | The root path of the workflow application.                                                                         |
 | **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             |   Yes    | List of importable string for the call stage.                                                                      |
 | **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    |   Yes    | List of importable string for the filter template.                                                                 |
-| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |    No    | The config path that keep all template `.yaml` files.                                                              |
+| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |   Yes    | The config path that keep all template `.yaml` files.                                                              |
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  |    No    | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
-| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |    No    | A flag that enable default stage ID that use for catch an execution output.                                        |
+| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |   Yes    | A flag that enable default stage ID that use for catch an execution output.                                        |
 | **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         |   Yes    | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |    No    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
+| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |   Yes    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
 | **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          |   Yes    | A flag that all job raise JobException from job strategy execution.                                                |
 | **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |    No    |                                                                                                                    |
 | **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |    No    |                                                                                                                    |

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/README.md

@@ -27,9 +27,9 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. The Minimum frequency unit of scheduling is **1 Minute** 🕘
+1. The Minimum frequency unit of built-in scheduling is **1 Minute** 🕘
 2. **Can not** re-run only failed stage and its pending downstream ↩️
-3. All parallel tasks inside workflow engine use **Multi-Threading**
+3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
    (Python 3.13 unlock GIL 🐍🔓)
 
 ---
@@ -222,11 +222,11 @@ it will use default value and do not raise any error to you.
 | **ROOT_PATH**                |   Core    | `.`                                                                                                                             |    No    | The root path of the workflow application.                                                                         |
 | **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             |   Yes    | List of importable string for the call stage.                                                                      |
 | **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    |   Yes    | List of importable string for the filter template.                                                                 |
-| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |    No    | The config path that keep all template `.yaml` files.                                                              |
+| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |   Yes    | The config path that keep all template `.yaml` files.                                                              |
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  |    No    | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
-| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |    No    | A flag that enable default stage ID that use for catch an execution output.                                        |
+| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |   Yes    | A flag that enable default stage ID that use for catch an execution output.                                        |
 | **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         |   Yes    | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |    No    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
+| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |   Yes    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
 | **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          |   Yes    | A flag that all job raise JobException from job strategy execution.                                                |
 | **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |    No    |                                                                                                                    |
 | **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |    No    |                                                                                                                    |

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/pyproject.toml

@@ -38,7 +38,6 @@ dynamic = ["version"]
 all = [
     "fastapi>=0.115.0,<1.0.0",
     "httpx",
-    "ujson",
     "aiofiles",
     "aiohttp",
 ]

ddeutil_workflow-0.0.45/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.45"

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil/workflow/conf.py

@@ -341,6 +341,32 @@ class SimLoad:
         )
 
 
+config: Config = Config()
+api_config: APIConfig = APIConfig()
+
+
+def dynamic(
+    key: Optional[str] = None,
+    *,
+    f: Optional[T] = None,
+    extras: Optional[DictData] = None,
+) -> Optional[T]:
+    """Dynamic get config if extra value was passed at run-time.
+
+    :param key: (str) A config key that get from Config object.
+    :param f: An inner config function scope.
+    :param extras: An extra values that pass at run-time.
+    """
+    rsx: Optional[T] = extras[key] if extras and key in extras else None
+    rs: Optional[T] = f or getattr(config, key, None)
+    if rsx is not None and not isinstance(rsx, type(rs)):
+        raise TypeError(
+            f"Type of config {key!r} from extras: {rsx!r} does not valid "
+            f"as config {type(rs)}."
+        )
+    return rsx or rs
+
+
 class Loader(SimLoad):
     """Loader Object that get the config `yaml` file from current path.
 
@@ -355,6 +381,7 @@ class Loader(SimLoad):
         *,
         included: list[str] | None = None,
         excluded: list[str] | None = None,
+        path: Path | None = None,
         **kwargs,
     ) -> Iterator[tuple[str, DictData]]:
         """Override the find class method from the Simple Loader object.
@@ -362,44 +389,23 @@ class Loader(SimLoad):
         :param obj: An object that want to validate matching before return.
         :param included:
         :param excluded:
+        :param path:
 
         :rtype: Iterator[tuple[str, DictData]]
         """
         return super().finds(
             obj=obj,
-            conf_path=config.conf_path,
+            conf_path=(path or config.conf_path),
             included=included,
             excluded=excluded,
         )
 
     def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, conf_path=config.conf_path, externals=externals)
-
-
-config: Config = Config()
-api_config: APIConfig = APIConfig()
-
-
-def dynamic(
-    key: Optional[str] = None,
-    *,
-    f: Optional[T] = None,
-    extras: Optional[DictData] = None,
-) -> Optional[T]:
-    """Dynamic get config if extra value was passed at run-time.
-
-    :param key: (str) A config key that get from Config object.
-    :param f: An inner config function scope.
-    :param extras: An extra values that pass at run-time.
-    """
-    rsx: Optional[T] = extras[key] if extras and key in extras else None
-    rs: Optional[T] = f or getattr(config, key, None)
-    if rsx is not None and not isinstance(rsx, type(rs)):
-        raise TypeError(
-            f"Type of config {key!r} from extras: {rsx!r} does not valid "
-            f"as config {type(rs)}."
+        super().__init__(
+            name,
+            conf_path=dynamic("conf_path", extras=externals),
+            externals=externals,
         )
-    return rsx or rs
 
 
 @lru_cache

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil/workflow/logs.py

@@ -80,6 +80,8 @@ class TraceMeda(BaseModel):  # pragma: no cov
 
 
 class TraceData(BaseModel):  # pragma: no cov
+    """Trace Data model for keeping data for any Trace models."""
+
     stdout: str = Field(description="A standard output trace data.")
     stderr: str = Field(description="A standard error trace data.")
     meta: list[TraceMeda] = Field(
@@ -92,6 +94,12 @@ class TraceData(BaseModel):  # pragma: no cov
 
     @classmethod
     def from_path(cls, file: Path) -> Self:
+        """Construct this trace data model with a trace path.
+
+        :param file: (Path) A trace path.
+
+        :rtype: Self
+        """
         data: DictStr = {"stdout": "", "stderr": "", "meta": []}
 
         if (file / "stdout.txt").exists():
@@ -207,27 +215,52 @@ class BaseTraceLog(ABC):  # pragma: no cov
         logger.exception(msg, stacklevel=2)
 
     async def adebug(self, message: str) -> None:  # pragma: no cov
+        """Async write trace log with append mode and logging this message with
+        the DEBUG level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         if config.debug:
             await self.awriter(msg)
         logger.info(msg, stacklevel=2)
 
     async def ainfo(self, message: str) -> None:  # pragma: no cov
+        """Async write trace log with append mode and logging this message with
+        the INFO level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         await self.awriter(msg)
         logger.info(msg, stacklevel=2)
 
     async def awarning(self, message: str) -> None:  # pragma: no cov
+        """Async write trace log with append mode and logging this message with
+        the WARNING level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         await self.awriter(msg)
         logger.warning(msg, stacklevel=2)
 
     async def aerror(self, message: str) -> None:  # pragma: no cov
+        """Async write trace log with append mode and logging this message with
+        the ERROR level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         await self.awriter(msg, is_err=True)
         logger.error(msg, stacklevel=2)
 
     async def aexception(self, message: str) -> None:  # pragma: no cov
+        """Async write trace log with append mode and logging this message with
+        the EXCEPTION level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         await self.awriter(msg, is_err=True)
         logger.exception(msg, stacklevel=2)
@@ -237,23 +270,29 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     """Trace Log object that write file to the local storage."""
 
     @classmethod
-    def find_logs(cls) -> Iterator[TraceData]:  # pragma: no cov
+    def find_logs(
+        cls, path: Path | None = None
+    ) -> Iterator[TraceData]:  # pragma: no cov
+        """Find trace logs."""
         for file in sorted(
-            config.log_path.glob("./run_id=*"),
+            (path or config.log_path).glob("./run_id=*"),
             key=lambda f: f.lstat().st_mtime,
         ):
             yield TraceData.from_path(file)
 
     @classmethod
     def find_log_with_id(
-        cls, run_id: str, force_raise: bool = True
+        cls, run_id: str, force_raise: bool = True, *, path: Path | None = None
     ) -> TraceData:
-        file: Path = config.log_path / f"run_id={run_id}"
+        """Find trace log with an input specific run ID."""
+        base_path: Path = path or config.log_path
+        file: Path = base_path / f"run_id={run_id}"
         if file.exists():
             return TraceData.from_path(file)
         elif force_raise:
             raise FileNotFoundError(
-                f"Trace log on path 'run_id={run_id}' does not found."
+                f"Trace log on path {base_path}, does not found trace "
+                f"'run_id={run_id}'."
             )
         return {}
 

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil/workflow/result.py

@@ -72,6 +72,7 @@ class Result:
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
     trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
+    extras: DictData = field(default_factory=dict)
 
     @classmethod
     def construct_with_rs_or_id(
@@ -80,6 +81,8 @@ class Result:
         run_id: str | None = None,
         parent_run_id: str | None = None,
         id_logic: str | None = None,
+        *,
+        extras: DictData | None = None,
     ) -> Self:
         """Create the Result object or set parent running id if passing Result
         object.
@@ -88,16 +91,22 @@ class Result:
         :param run_id:
         :param parent_run_id:
         :param id_logic:
+        :param extras:
 
         :rtype: Self
         """
         if result is None:
-            result: Result = cls(
+            return cls(
                 run_id=(run_id or gen_id(id_logic or "", unique=True)),
                 parent_run_id=parent_run_id,
+                extras=(extras or {}),
             )
         elif parent_run_id:
             result.set_parent_run_id(parent_run_id)
+
+        if extras is not None:
+            result.extras.update(extras)
+
         return result
 
     @model_validator(mode="after")

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil/workflow/reusables.py

@@ -499,6 +499,7 @@ class CallSearchData:
 
 def extract_call(
     call: str,
+    *,
     registries: Optional[list[str]] = None,
 ) -> Callable[[], TagFunc]:
     """Extract Call function from string value to call partial function that

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil/workflow/workflow.py

@@ -314,7 +314,8 @@ class Workflow(BaseModel):
 
         loader_data: DictData = copy.deepcopy(loader.data)
         loader_data["name"] = name.replace(" ", "_")
-        if extras:  # pragma: no cov
+
+        if extras:
             loader_data["extras"] = extras
 
         cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
@@ -325,6 +326,7 @@ class Workflow(BaseModel):
         cls,
         name: str,
         path: Path,
+        *,
         extras: DictData | None = None,
     ) -> Self:
         """Create Workflow instance from the specific path. The loader object
@@ -349,7 +351,8 @@ class Workflow(BaseModel):
 
         loader_data: DictData = copy.deepcopy(loader.data)
         loader_data["name"] = name.replace(" ", "_")
-        if extras:  # pragma: no cov
+
+        if extras:
             loader_data["extras"] = extras
 
         cls.__bypass_on__(loader_data, path=path, extras=extras)

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil_workflow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.44
+Version: 0.0.45
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -30,7 +30,6 @@ Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: all
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
 Requires-Dist: httpx; extra == "all"
-Requires-Dist: ujson; extra == "all"
 Requires-Dist: aiofiles; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Provides-Extra: api
@@ -71,9 +70,9 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. The Minimum frequency unit of scheduling is **1 Minute** 🕘
+1. The Minimum frequency unit of built-in scheduling is **1 Minute** 🕘
 2. **Can not** re-run only failed stage and its pending downstream ↩️
-3. All parallel tasks inside workflow engine use **Multi-Threading**
+3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
    (Python 3.13 unlock GIL 🐍🔓)
 
 ---
@@ -266,11 +265,11 @@ it will use default value and do not raise any error to you.
 | **ROOT_PATH**                |   Core    | `.`                                                                                                                             |    No    | The root path of the workflow application.                                                                         |
 | **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             |   Yes    | List of importable string for the call stage.                                                                      |
 | **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    |   Yes    | List of importable string for the filter template.                                                                 |
-| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |    No    | The config path that keep all template `.yaml` files.                                                              |
+| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |   Yes    | The config path that keep all template `.yaml` files.                                                              |
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  |    No    | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
-| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |    No    | A flag that enable default stage ID that use for catch an execution output.                                        |
+| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |   Yes    | A flag that enable default stage ID that use for catch an execution output.                                        |
 | **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         |   Yes    | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |    No    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
+| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |   Yes    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
 | **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          |   Yes    | A flag that all job raise JobException from job strategy execution.                                                |
 | **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |    No    |                                                                                                                    |
 | **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |    No    |                                                                                                                    |

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/src/ddeutil_workflow.egg-info/requires.txt

@@ -7,7 +7,6 @@ schedule<2.0.0,==1.2.2
 [all]
 fastapi<1.0.0,>=0.115.0
 httpx
-ujson
 aiofiles
 aiohttp
 

{ddeutil_workflow-0.0.44 → ddeutil_workflow-0.0.45}/tests/test_workflow.py

@@ -1,3 +1,6 @@
+import shutil
+from pathlib import Path
+
 import pytest
 from ddeutil.workflow import Workflow
 from ddeutil.workflow.exceptions import WorkflowException
@@ -151,6 +154,54 @@ def test_workflow_from_path(test_path):
     assert rs.context == {}
 
 
+def test_workflow_from_loader_override(test_path):
+    conf_path: Path = test_path / "mock_conf"
+    conf_path.mkdir(exist_ok=True)
+    (conf_path / "demo").mkdir(exist_ok=True)
+
+    with dump_yaml_context(
+        conf_path / "demo/01_99_wf_test_override_config.yml",
+        data="""
+        tmp-wf-override-conf:
+          type: Workflow
+          param: {name: str}
+          jobs:
+            first-job:
+              stages:
+                - name: "Hello"
+                  echo: "Hello ${{ params.name }}"
+
+        tmp-wf-override-conf-trigger:
+          type: Workflow
+          params: {name: str}
+          jobs:
+            trigger-job:
+              stages:
+                - name: "Trigger override"
+                  id: trigger-stage
+                  trigger: tmp-wf-override-conf
+                  params:
+                    name: ${{ params.name }}
+        """,
+    ):
+        workflow = Workflow.from_conf(
+            name="tmp-wf-override-conf", extras={"conf_path": conf_path}
+        )
+        rs: Result = workflow.execute(params={"name": "foo"})
+        print(rs.context)
+
+        workflow = Workflow.from_conf(
+            name="tmp-wf-override-conf-trigger", extras={"conf_path": conf_path}
+        )
+        stage = workflow.job(name="trigger-job").stage("trigger-stage")
+        assert stage.extras == {"conf_path": conf_path}
+
+        rs: Result = workflow.execute(params={"name": "bar"})
+        print(rs.context)
+
+    shutil.rmtree(conf_path)
+
+
 def test_workflow_from_loader_raise(test_path):
     test_file = test_path / "conf/demo/01_01_wf_run_raise.yml"
 

ddeutil_workflow-0.0.44/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.44"