ddeutil-workflow 0.0.45__tar.gz → 0.0.47__tar.gz

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.45
+Version: 0.0.47
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/pyproject.toml

@@ -13,7 +13,6 @@ classifiers = [
     "Topic :: Utilities",
     "Natural Language :: English",
     "Development Status :: 4 - Beta",
-    # "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
     "Programming Language :: Python",

ddeutil_workflow-0.0.47/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.47"

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/conf.py

@@ -26,7 +26,10 @@ PREFIX: str = "WORKFLOW"
 
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
-    """Get environment variable with uppercase and adding prefix string."""
+    """Get environment variable with uppercase and adding prefix string.
+
+    :rtype: str | None
+    """
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
@@ -42,29 +45,7 @@ __all__: TupleStr = (
 )
 
 
-class BaseConfig:  # pragma: no cov
-    """BaseConfig object inheritable."""
-
-    __slots__ = ()
-
-    @property
-    def root_path(self) -> Path:
-        """Root path or the project path.
-
-        :rtype: Path
-        """
-        return Path(os.getenv("ROOT_PATH", "."))
-
-    @property
-    def conf_path(self) -> Path:
-        """Config path that use root_path class argument for this construction.
-
-        :rtype: Path
-        """
-        return self.root_path / os.getenv("CONF_PATH", "conf")
-
-
-class Config(BaseConfig):  # pragma: no cov
+class Config:  # pragma: no cov
     """Config object for keeping core configurations on the current session
     without changing when if the application still running.
 
@@ -217,6 +198,7 @@ class Config(BaseConfig):  # pragma: no cov
 
 
 class APIConfig:
+    """API Config object."""
 
     @property
     def prefix_path(self) -> str:

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/job.py

@@ -641,6 +641,9 @@ def local_execute_strategy(
 
     for stage in job.stages:
 
+        if job.extras:
+            stage.extras = job.extras
+
         if stage.is_skipped(params=context):
             result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
             stage.set_outputs(output={"skipped": True}, to=context)

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/result.py

@@ -18,19 +18,10 @@ from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 from .logs import TraceLog, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id
 
-__all__: TupleStr = (
-    "SUCCESS",
-    "FAILED",
-    "WAIT",
-    "SKIP",
-    "Result",
-    "Status",
-)
-
 
 class Status(IntEnum):
     """Status Int Enum object that use for tracking execution status to the
@@ -62,6 +53,10 @@ class Result:
 
         For comparison property, this result will use ``status``, ``context``,
     and ``_run_id`` fields to comparing with other result instance.
+
+    Warning:
+        I use dataclass object instead of Pydantic model object because context
+    field that keep dict value change its ID when update new value to it.
     """
 
     status: Status = field(default=WAIT)
@@ -87,11 +82,11 @@ class Result:
         """Create the Result object or set parent running id if passing Result
         object.
 
-        :param result:
-        :param run_id:
-        :param parent_run_id:
-        :param id_logic:
-        :param extras:
+        :param result: A Result instance.
+        :param run_id: A running ID.
+        :param parent_run_id: A parent running ID.
+        :param id_logic: A logic function that use to generate a running ID.
+        :param extras: An extra parameter that want to override the core config.
 
         :rtype: Self
         """

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/reusables.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
-from dataclasses import dataclass
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
@@ -23,6 +22,7 @@ except ImportError:
 
 from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
+from pydantic.dataclasses import dataclass
 
 from .__types import DictData, Re
 from .conf import dynamic
@@ -437,6 +437,7 @@ Registry = dict[str, Callable[[], TagFunc]]
 
 def make_registry(
     submodule: str,
+    *,
     registries: Optional[list[str]] = None,
 ) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
@@ -539,13 +540,13 @@ def extract_call(
 
     if call.func not in rgt:
         raise NotImplementedError(
-            f"`REGISTER-MODULES.{call.path}.registries` not implement "
+            f"`REGISTERS.{call.path}.registries` not implement "
             f"registry: {call.func!r}."
         )
 
     if call.tag not in rgt[call.func]:
         raise NotImplementedError(
             f"tag: {call.tag!r} not found on registry func: "
-            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
+            f"`REGISTER.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/stages.py

@@ -38,12 +38,13 @@ from concurrent.futures import (
     ThreadPoolExecutor,
     as_completed,
 )
+from dataclasses import is_dataclass
 from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Optional, Union
+from typing import Annotated, Any, Optional, Union, get_type_hints
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
@@ -814,7 +815,8 @@ class CallStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        t_func: TagFunc = extract_call(
+        has_keyword: bool = False
+        call_func: TagFunc = extract_call(
             param2template(self.uses, params, extras=self.extras),
             registries=self.extras.get("regis_call"),
         )()
@@ -824,55 +826,92 @@ class CallStage(BaseStage):
         args: DictData = {"result": result} | param2template(
             self.args, params, extras=self.extras
         )
-        ips = inspect.signature(t_func)
-        necessary_params: list[str] = [
-            k
-            for k in ips.parameters
+        ips = inspect.signature(call_func)
+        necessary_params: list[str] = []
+        for k in ips.parameters:
             if (
-                (v := ips.parameters[k]).default == Parameter.empty
-                and (
-                    v.kind != Parameter.VAR_KEYWORD
-                    or v.kind != Parameter.VAR_POSITIONAL
-                )
-            )
-        ]
+                v := ips.parameters[k]
+            ).default == Parameter.empty and v.kind not in (
+                Parameter.VAR_KEYWORD,
+                Parameter.VAR_POSITIONAL,
+            ):
+                necessary_params.append(k)
+            elif v.kind == Parameter.VAR_KEYWORD:
+                has_keyword = True
+
         if any(
             (k.removeprefix("_") not in args and k not in args)
             for k in necessary_params
         ):
             raise ValueError(
                 f"Necessary params, ({', '.join(necessary_params)}, ), "
-                f"does not set to args"
+                f"does not set to args, {list(args.keys())}."
             )
 
-        # NOTE: add '_' prefix if it wants to use.
-        for k in ips.parameters:
-            if k.removeprefix("_") in args:
-                args[k] = args.pop(k.removeprefix("_"))
-
-        if "result" not in ips.parameters:
+        if "result" not in ips.parameters and not has_keyword:
             args.pop("result")
 
-        result.trace.info(f"[STAGE]: Call-Execute: {t_func.name}@{t_func.tag}")
-        if inspect.iscoroutinefunction(t_func):  # pragma: no cov
+        result.trace.info(
+            f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
+        )
+
+        args = self.parse_model_args(call_func, args, result)
+        if inspect.iscoroutinefunction(call_func):
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
-                t_func(**param2template(args, params, extras=self.extras))
+                call_func(**param2template(args, params, extras=self.extras))
             )
         else:
-            rs: DictData = t_func(
+            rs: DictData = call_func(
                 **param2template(args, params, extras=self.extras)
             )
 
         # VALIDATE:
         #   Check the result type from call function, it should be dict.
-        if not isinstance(rs, dict):
+        if isinstance(rs, BaseModel):
+            rs: DictData = rs.model_dump()
+        elif not isinstance(rs, dict):
             raise TypeError(
-                f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
-                f"to result model, you change return type to `dict`."
+                f"Return type: '{call_func.name}@{call_func.tag}' does not "
+                f"serialize to result model, you change return type to `dict`."
             )
         return result.catch(status=SUCCESS, context=rs)
 
+    @staticmethod
+    def parse_model_args(
+        func: TagFunc,
+        args: DictData,
+        result: Result,
+    ) -> DictData:
+        """Parse Pydantic model from any dict data before parsing to target
+        caller function.
+        """
+        try:
+            type_hints: dict[str, Any] = get_type_hints(func)
+        except TypeError as e:
+            result.trace.warning(
+                f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
+                f"parsing model args process."
+            )
+            return args
+
+        for arg in type_hints:
+
+            if arg == "return":
+                continue
+
+            if arg.removeprefix("_") in args:
+                args[arg] = args.pop(arg.removeprefix("_"))
+
+            t: Any = type_hints[arg]
+            if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
+                args[arg] = result
+
+            if issubclass(t, BaseModel) and arg in args:
+                args[arg] = t.model_validate(obj=args[arg])
+
+        return args
+
 
 class TriggerStage(BaseStage):
     """Trigger Workflow execution stage that execute another workflow. This
@@ -976,6 +1015,8 @@ class ParallelStage(BaseStage):  # pragma: no cov
         params: DictData,
         result: Result,
         stages: list[Stage],
+        *,
+        extras: DictData | None = None,
     ) -> DictData:
         """Task execution method for passing a branch to each thread.
 
@@ -984,12 +1025,16 @@ class ParallelStage(BaseStage):  # pragma: no cov
         :param result: (Result) A result object for keeping context and status
             data.
         :param stages:
+        :param extras
 
         :rtype: DictData
         """
         context = {"branch": branch, "stages": {}}
         result.trace.debug(f"[STAGE]: Execute parallel branch: {branch!r}")
         for stage in stages:
+            if extras:
+                stage.extras = extras
+
             try:
                 stage.set_outputs(
                     stage.handler_execute(
@@ -1048,6 +1093,7 @@ class ParallelStage(BaseStage):  # pragma: no cov
                         params=params,
                         result=result,
                         stages=self.parallel[branch],
+                        extras=self.extras,
                     )
                 )
 
@@ -1144,6 +1190,10 @@ class ForEachStage(BaseStage):
             context = {"stages": {}}
 
             for stage in self.stages:
+
+                if self.extras:
+                    stage.extras = self.extras
+
                 try:
                     stage.set_outputs(
                         stage.handler_execute(
@@ -1284,6 +1334,9 @@ class CaseStage(BaseStage):  # pragma: no cov
 
             if match == _condition:
                 stage: Stage = match.stage
+                if self.extras:
+                    stage.extras = self.extras
+
                 try:
                     stage.set_outputs(
                         stage.handler_execute(

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/workflow.py

@@ -1139,7 +1139,7 @@ class Workflow(BaseModel):
                 not_timeout_flag := ((time.monotonic() - ts) < timeout)
             ):
                 job_id: str = job_queue.get()
-                job: Job = self.jobs[job_id]
+                job: Job = self.job(name=job_id)
 
                 if (check := job.check_needs(context["jobs"])) == WAIT:
                     job_queue.task_done()
@@ -1231,7 +1231,7 @@ class Workflow(BaseModel):
                 not_timeout_flag := ((time.monotonic() - ts) < timeout)
             ):
                 job_id: str = job_queue.get()
-                job: Job = self.jobs[job_id]
+                job: Job = self.job(name=job_id)
 
                 if (check := job.check_needs(context["jobs"])) == WAIT:
                     job_queue.task_done()

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil_workflow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.45
+Version: 0.0.47
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/src/ddeutil_workflow.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ pyproject.toml
 src/ddeutil/workflow/__about__.py
 src/ddeutil/workflow/__cron.py
 src/ddeutil/workflow/__init__.py
+src/ddeutil/workflow/__main__.py
 src/ddeutil/workflow/__types.py
 src/ddeutil/workflow/conf.py
 src/ddeutil/workflow/cron.py

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/tests/test_reusables_call_tag.py

@@ -3,11 +3,14 @@ from __future__ import annotations
 import asyncio
 import inspect
 import shutil
+import typing
+from dataclasses import is_dataclass
 from pathlib import Path
 from textwrap import dedent
 
 import pytest
-from ddeutil.workflow.reusables import Registry, make_registry
+from ddeutil.workflow.reusables import Registry, extract_call, make_registry
+from pydantic import BaseModel
 
 
 @pytest.fixture(scope="module")
@@ -94,10 +97,10 @@ def test_make_registry(call_function):
 
 def test_make_registry_from_env():
     rs: dict[str, Registry] = make_registry("tasks")
-    print(rs)
     assert set(rs.keys()) == {
         "async-el-csv-to-parquet",
         "get-items",
+        "gen-type",
         "mssql-proc",
         "el-csv-to-parquet",
         "return-type-not-valid",
@@ -116,12 +119,36 @@ def test_make_registry_raise(call_function_dup):
         make_registry("new_tasks_dup")
 
 
+def test_extract_call():
+    func = extract_call("tasks/el-csv-to-parquet@polars-dir")
+    call_func = func()
+    assert call_func.name == "el-csv-to-parquet"
+    assert call_func.tag == "polars-dir"
+
+
+def test_extract_call_args_type():
+    func = extract_call("tasks/gen-type@demo")
+    call_func = func()
+
+    get_types = typing.get_type_hints(call_func)
+    for p in get_types:
+        if is_dataclass(get_types[p]) and get_types[p].__name__ == "Result":
+            print("found result", p, get_types[p])
+        if issubclass(get_types[p], BaseModel):
+            print(p, "with type:", get_types[p])
+
+
 @pytest.mark.skip("Skip because it uses for local test only.")
 def test_inspec_func():
 
     def demo_func(
         args_1: str, args_2: Path, *args, kwargs_1: str | None = None, **kwargs
     ):  # pragma: no cov
+        _ = args_1
+        _ = args_2
+        _ = args
+        _ = kwargs_1
+        _ = kwargs
         pass
 
     assert inspect.isfunction(demo_func)
@@ -141,6 +168,11 @@ def test_inspec_func():
         args_1: str, args_2: Path, *args, kwargs_1: str | None = None, **kwargs
     ):  # pragma: no cov
         await asyncio.sleep(0.1)
+        _ = args_1
+        _ = args_2
+        _ = args
+        _ = kwargs_1
+        _ = kwargs
         pass
 
     print(inspect.isfunction(ademo_func))
@@ -158,3 +190,61 @@ def test_inspec_func():
     #     print(v.default)
     #     print(v.kind, " (", type(v.kind), ")")
     #     print("-----")
+
+
+class MockModel(BaseModel):  # pragma: no cov
+    name: str
+
+
+def outside_func(args: MockModel) -> MockModel:  # pragma: no cov
+    _ = args
+    pass
+
+
+@pytest.mark.skip("Skip because it uses for local test only.")
+def test_inspec_with_pydantic_model_args():
+    from pydantic import BaseModel
+
+    class MockModelLocal(BaseModel):
+        name: str
+
+    def demo_func(
+        args_1: MockModel,
+        args_2: MockModelLocal,
+        *args,
+        kwargs_1: str = None,
+        **kwargs,
+    ) -> MockModel:  # pragma: no cov
+        _ = args_1
+        _ = args_2
+        _ = args
+        _ = kwargs_1
+        _ = kwargs
+        pass
+
+    # ips = inspect.signature(demo_func)
+    # for k, v in ips.parameters.items():
+    #     print(k)
+    #     print(ips.parameters[k].default)
+    #     print(v)
+    #     print(v.name)
+    #     print(v.annotation, "type:", type(v.annotation))
+    #     print(v.default)
+    #     print(v.kind, " (", type(v.kind), ")")
+    #     print("-----")
+    #
+    # print(ips.return_annotation, "type:", type(ips.return_annotation))
+    # print(ips.return_annotation is MockModel)
+    # print(ips.return_annotation.__parameter__)
+
+    import typing
+
+    rs = typing.get_type_hints(demo_func, localns=locals(), globalns=globals())
+    print(rs)
+
+    print(demo_func.__annotations__)
+    print(globals()["MockModel"])
+    print(locals()["MockModelLocal"])
+
+    rs = typing.get_type_hints(outside_func)
+    print(rs)

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/tests/test_stage_handler_exec.py

@@ -97,20 +97,29 @@ def test_stage_exec_call(test_path):
                   with:
                     source: src
                     sink: sink
+                - name: "Return with Pydantic Model"
+                  id: return-model
+                  uses: tasks/gen-type@demo
+                  with:
+                    args1: foo
+                    args2: conf/path
+                    args3:
+                      name: test
+                      data:
+                        input: hello
         """,
     ):
         workflow = Workflow.from_conf(name="tmp-wf-call-return-type")
 
         stage: Stage = workflow.job("second-job").stage("extract-load")
         rs: Result = stage.handler_execute({})
-        print(rs)
+        assert 0 == rs.status
+        assert {"records": 1} == rs.context
 
         stage: Stage = workflow.job("second-job").stage("async-extract-load")
         rs: Result = stage.handler_execute({})
-        print(rs)
-
-        assert 0 == rs.status
-        assert {"records": 1} == rs.context
+        assert rs.status == 0
+        assert rs.context == {"records": 1}
 
         # NOTE: Raise because invalid return type.
         with pytest.raises(StageException):
@@ -132,6 +141,11 @@ def test_stage_exec_call(test_path):
             stage: Stage = workflow.job("first-job").stage("call-not-register")
             stage.handler_execute({})
 
+        stage: Stage = workflow.job("second-job").stage("return-model")
+        rs: Result = stage.handler_execute({})
+        assert rs.status == 0
+        assert rs.context == {"name": "foo", "data": {"key": "value"}}
+
 
 @mock.patch.object(Config, "stage_raise_error", True)
 def test_stage_exec_py_raise():

{ddeutil_workflow-0.0.45 → ddeutil_workflow-0.0.47}/tests/test_workflow_exec.py

@@ -1,8 +1,10 @@
+import shutil
 from datetime import datetime
+from textwrap import dedent
 from unittest import mock
 
 from ddeutil.core import getdot
-from ddeutil.workflow import SUCCESS, Workflow
+from ddeutil.workflow import SUCCESS, Workflow, extract_call
 from ddeutil.workflow.conf import Config
 from ddeutil.workflow.job import Job
 from ddeutil.workflow.result import FAILED, Result
@@ -488,6 +490,55 @@ def test_workflow_exec_call(test_path):
         } == rs.context
 
 
+def test_workflow_exec_call_override_registry(test_path):
+    task_path = test_path.parent / "mock_tests"
+    task_path.mkdir(exist_ok=True)
+    (task_path / "__init__.py").open(mode="w")
+    (task_path / "mock_tasks").mkdir(exist_ok=True)
+
+    with (task_path / "mock_tasks/__init__.py").open(mode="w") as f:
+        f.write(
+            dedent(
+                """
+            from ddeutil.workflow import tag, Result
+
+            @tag("v1", alias="get-info")
+            def get_info(result: Result):
+                result.trace.info("... [CALLER]: Info from mock tasks")
+                return {"get-info": "success"}
+
+            """.strip(
+                    "\n"
+                )
+            )
+        )
+
+    with dump_yaml_context(
+        test_path / "conf/demo/01_99_wf_test_wf_exec_call_override.yml",
+        data="""
+        tmp-wf-exec-call-override:
+          type: Workflow
+          jobs:
+            first-job:
+              stages:
+                - name: "Call from mock tasks"
+                  uses: mock_tasks/get-info@v1
+        """,
+    ):
+        func = extract_call("mock_tasks/get-info@v1", registries=["mock_tests"])
+        assert func().name == "get-info"
+
+        workflow = Workflow.from_conf(
+            name="tmp-wf-exec-call-override",
+            extras={"regis_call": ["mock_tests"]},
+        )
+        rs = workflow.execute(params={})
+        assert rs.status == SUCCESS
+        print(rs.context)
+
+    shutil.rmtree(task_path)
+
+
 def test_workflow_exec_call_with_prefix(test_path):
     with dump_yaml_context(
         test_path / "conf/demo/01_99_wf_test_wf_call_mssql_proc.yml",

ddeutil_workflow-0.0.45/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.45"