ddeutil-workflow 0.0.46__tar.gz → 0.0.47__tar.gz

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.46
+Version: 0.0.47
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -9,7 +9,7 @@ Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
 Keywords: orchestration,workflow
 Classifier: Topic :: Utilities
 Classifier: Natural Language :: English
-Classifier: Development Status :: 5 - Production/Stable
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/pyproject.toml

@@ -12,7 +12,7 @@ keywords = ['orchestration', 'workflow']
 classifiers = [
     "Topic :: Utilities",
     "Natural Language :: English",
-    "Development Status :: 5 - Production/Stable",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
     "Programming Language :: Python",

ddeutil_workflow-0.0.47/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.47"

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/result.py

@@ -18,19 +18,10 @@ from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 from .logs import TraceLog, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id
 
-__all__: TupleStr = (
-    "SUCCESS",
-    "FAILED",
-    "WAIT",
-    "SKIP",
-    "Result",
-    "Status",
-)
-
 
 class Status(IntEnum):
     """Status Int Enum object that use for tracking execution status to the
@@ -62,6 +53,10 @@ class Result:
 
         For comparison property, this result will use ``status``, ``context``,
     and ``_run_id`` fields to comparing with other result instance.
+
+    Warning:
+        I use dataclass object instead of Pydantic model object because context
+    field that keep dict value change its ID when update new value to it.
     """
 
     status: Status = field(default=WAIT)
@@ -87,11 +82,11 @@ class Result:
         """Create the Result object or set parent running id if passing Result
         object.
 
-        :param result:
-        :param run_id:
-        :param parent_run_id:
-        :param id_logic:
-        :param extras:
+        :param result: A Result instance.
+        :param run_id: A running ID.
+        :param parent_run_id: A parent running ID.
+        :param id_logic: A logic function that use to generate a running ID.
+        :param extras: An extra parameter that want to override the core config.
 
         :rtype: Self
         """

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/reusables.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
-from dataclasses import dataclass
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
@@ -23,6 +22,7 @@ except ImportError:
 
 from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
+from pydantic.dataclasses import dataclass
 
 from .__types import DictData, Re
 from .conf import dynamic
@@ -437,6 +437,7 @@ Registry = dict[str, Callable[[], TagFunc]]
 
 def make_registry(
     submodule: str,
+    *,
     registries: Optional[list[str]] = None,
 ) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
@@ -539,13 +540,13 @@ def extract_call(
 
     if call.func not in rgt:
         raise NotImplementedError(
-            f"`REGISTER-MODULES.{call.path}.registries` not implement "
+            f"`REGISTERS.{call.path}.registries` not implement "
             f"registry: {call.func!r}."
         )
 
     if call.tag not in rgt[call.func]:
         raise NotImplementedError(
             f"tag: {call.tag!r} not found on registry func: "
-            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
+            f"`REGISTER.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/src/ddeutil/workflow/stages.py

@@ -38,12 +38,13 @@ from concurrent.futures import (
     ThreadPoolExecutor,
     as_completed,
 )
+from dataclasses import is_dataclass
 from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Optional, Union
+from typing import Annotated, Any, Optional, Union, get_type_hints
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
@@ -814,9 +815,8 @@ class CallStage(BaseStage):
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
             )
 
-        print("Extras in CallStage", self.extras)
-
-        t_func: TagFunc = extract_call(
+        has_keyword: bool = False
+        call_func: TagFunc = extract_call(
             param2template(self.uses, params, extras=self.extras),
             registries=self.extras.get("regis_call"),
         )()
@@ -826,55 +826,92 @@ class CallStage(BaseStage):
         args: DictData = {"result": result} | param2template(
             self.args, params, extras=self.extras
         )
-        ips = inspect.signature(t_func)
-        necessary_params: list[str] = [
-            k
-            for k in ips.parameters
+        ips = inspect.signature(call_func)
+        necessary_params: list[str] = []
+        for k in ips.parameters:
             if (
-                (v := ips.parameters[k]).default == Parameter.empty
-                and (
-                    v.kind != Parameter.VAR_KEYWORD
-                    or v.kind != Parameter.VAR_POSITIONAL
-                )
-            )
-        ]
+                v := ips.parameters[k]
+            ).default == Parameter.empty and v.kind not in (
+                Parameter.VAR_KEYWORD,
+                Parameter.VAR_POSITIONAL,
+            ):
+                necessary_params.append(k)
+            elif v.kind == Parameter.VAR_KEYWORD:
+                has_keyword = True
+
         if any(
             (k.removeprefix("_") not in args and k not in args)
             for k in necessary_params
         ):
             raise ValueError(
                 f"Necessary params, ({', '.join(necessary_params)}, ), "
-                f"does not set to args"
+                f"does not set to args, {list(args.keys())}."
             )
 
-        # NOTE: add '_' prefix if it wants to use.
-        for k in ips.parameters:
-            if k.removeprefix("_") in args:
-                args[k] = args.pop(k.removeprefix("_"))
-
-        if "result" not in ips.parameters:
+        if "result" not in ips.parameters and not has_keyword:
             args.pop("result")
 
-        result.trace.info(f"[STAGE]: Call-Execute: {t_func.name}@{t_func.tag}")
-        if inspect.iscoroutinefunction(t_func):  # pragma: no cov
+        result.trace.info(
+            f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
+        )
+
+        args = self.parse_model_args(call_func, args, result)
+        if inspect.iscoroutinefunction(call_func):
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
-                t_func(**param2template(args, params, extras=self.extras))
+                call_func(**param2template(args, params, extras=self.extras))
             )
         else:
-            rs: DictData = t_func(
+            rs: DictData = call_func(
                 **param2template(args, params, extras=self.extras)
             )
 
         # VALIDATE:
         #   Check the result type from call function, it should be dict.
-        if not isinstance(rs, dict):
+        if isinstance(rs, BaseModel):
+            rs: DictData = rs.model_dump()
+        elif not isinstance(rs, dict):
             raise TypeError(
-                f"Return type: '{t_func.name}@{t_func.tag}' does not serialize "
-                f"to result model, you change return type to `dict`."
+                f"Return type: '{call_func.name}@{call_func.tag}' does not "
+                f"serialize to result model, you change return type to `dict`."
             )
         return result.catch(status=SUCCESS, context=rs)
 
+    @staticmethod
+    def parse_model_args(
+        func: TagFunc,
+        args: DictData,
+        result: Result,
+    ) -> DictData:
+        """Parse Pydantic model from any dict data before parsing to target
+        caller function.
+        """
+        try:
+            type_hints: dict[str, Any] = get_type_hints(func)
+        except TypeError as e:
+            result.trace.warning(
+                f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
+                f"parsing model args process."
+            )
+            return args
+
+        for arg in type_hints:
+
+            if arg == "return":
+                continue
+
+            if arg.removeprefix("_") in args:
+                args[arg] = args.pop(arg.removeprefix("_"))
+
+            t: Any = type_hints[arg]
+            if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
+                args[arg] = result
+
+            if issubclass(t, BaseModel) and arg in args:
+                args[arg] = t.model_validate(obj=args[arg])
+
+        return args
+
 
 class TriggerStage(BaseStage):
     """Trigger Workflow execution stage that execute another workflow. This

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/src/ddeutil_workflow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.46
+Version: 0.0.47
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -9,7 +9,7 @@ Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
 Keywords: orchestration,workflow
 Classifier: Topic :: Utilities
 Classifier: Natural Language :: English
-Classifier: Development Status :: 5 - Production/Stable
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/tests/test_reusables_call_tag.py

@@ -3,11 +3,14 @@ from __future__ import annotations
 import asyncio
 import inspect
 import shutil
+import typing
+from dataclasses import is_dataclass
 from pathlib import Path
 from textwrap import dedent
 
 import pytest
-from ddeutil.workflow.reusables import Registry, make_registry
+from ddeutil.workflow.reusables import Registry, extract_call, make_registry
+from pydantic import BaseModel
 
 
 @pytest.fixture(scope="module")
@@ -94,10 +97,10 @@ def test_make_registry(call_function):
 
 def test_make_registry_from_env():
     rs: dict[str, Registry] = make_registry("tasks")
-    print(rs)
     assert set(rs.keys()) == {
         "async-el-csv-to-parquet",
         "get-items",
+        "gen-type",
         "mssql-proc",
         "el-csv-to-parquet",
         "return-type-not-valid",
@@ -116,12 +119,36 @@ def test_make_registry_raise(call_function_dup):
         make_registry("new_tasks_dup")
 
 
+def test_extract_call():
+    func = extract_call("tasks/el-csv-to-parquet@polars-dir")
+    call_func = func()
+    assert call_func.name == "el-csv-to-parquet"
+    assert call_func.tag == "polars-dir"
+
+
+def test_extract_call_args_type():
+    func = extract_call("tasks/gen-type@demo")
+    call_func = func()
+
+    get_types = typing.get_type_hints(call_func)
+    for p in get_types:
+        if is_dataclass(get_types[p]) and get_types[p].__name__ == "Result":
+            print("found result", p, get_types[p])
+        if issubclass(get_types[p], BaseModel):
+            print(p, "with type:", get_types[p])
+
+
 @pytest.mark.skip("Skip because it uses for local test only.")
 def test_inspec_func():
 
     def demo_func(
         args_1: str, args_2: Path, *args, kwargs_1: str | None = None, **kwargs
     ):  # pragma: no cov
+        _ = args_1
+        _ = args_2
+        _ = args
+        _ = kwargs_1
+        _ = kwargs
         pass
 
     assert inspect.isfunction(demo_func)
@@ -141,6 +168,11 @@ def test_inspec_func():
         args_1: str, args_2: Path, *args, kwargs_1: str | None = None, **kwargs
     ):  # pragma: no cov
         await asyncio.sleep(0.1)
+        _ = args_1
+        _ = args_2
+        _ = args
+        _ = kwargs_1
+        _ = kwargs
         pass
 
     print(inspect.isfunction(ademo_func))
@@ -158,3 +190,61 @@ def test_inspec_func():
     #     print(v.default)
     #     print(v.kind, " (", type(v.kind), ")")
     #     print("-----")
+
+
+class MockModel(BaseModel):  # pragma: no cov
+    name: str
+
+
+def outside_func(args: MockModel) -> MockModel:  # pragma: no cov
+    _ = args
+    pass
+
+
+@pytest.mark.skip("Skip because it uses for local test only.")
+def test_inspec_with_pydantic_model_args():
+    from pydantic import BaseModel
+
+    class MockModelLocal(BaseModel):
+        name: str
+
+    def demo_func(
+        args_1: MockModel,
+        args_2: MockModelLocal,
+        *args,
+        kwargs_1: str = None,
+        **kwargs,
+    ) -> MockModel:  # pragma: no cov
+        _ = args_1
+        _ = args_2
+        _ = args
+        _ = kwargs_1
+        _ = kwargs
+        pass
+
+    # ips = inspect.signature(demo_func)
+    # for k, v in ips.parameters.items():
+    #     print(k)
+    #     print(ips.parameters[k].default)
+    #     print(v)
+    #     print(v.name)
+    #     print(v.annotation, "type:", type(v.annotation))
+    #     print(v.default)
+    #     print(v.kind, " (", type(v.kind), ")")
+    #     print("-----")
+    #
+    # print(ips.return_annotation, "type:", type(ips.return_annotation))
+    # print(ips.return_annotation is MockModel)
+    # print(ips.return_annotation.__parameter__)
+
+    import typing
+
+    rs = typing.get_type_hints(demo_func, localns=locals(), globalns=globals())
+    print(rs)
+
+    print(demo_func.__annotations__)
+    print(globals()["MockModel"])
+    print(locals()["MockModelLocal"])
+
+    rs = typing.get_type_hints(outside_func)
+    print(rs)

{ddeutil_workflow-0.0.46 → ddeutil_workflow-0.0.47}/tests/test_stage_handler_exec.py

@@ -97,20 +97,29 @@ def test_stage_exec_call(test_path):
                   with:
                     source: src
                     sink: sink
+                - name: "Return with Pydantic Model"
+                  id: return-model
+                  uses: tasks/gen-type@demo
+                  with:
+                    args1: foo
+                    args2: conf/path
+                    args3:
+                      name: test
+                      data:
+                        input: hello
         """,
     ):
         workflow = Workflow.from_conf(name="tmp-wf-call-return-type")
 
         stage: Stage = workflow.job("second-job").stage("extract-load")
         rs: Result = stage.handler_execute({})
-        print(rs)
+        assert 0 == rs.status
+        assert {"records": 1} == rs.context
 
         stage: Stage = workflow.job("second-job").stage("async-extract-load")
         rs: Result = stage.handler_execute({})
-        print(rs)
-
-        assert 0 == rs.status
-        assert {"records": 1} == rs.context
+        assert rs.status == 0
+        assert rs.context == {"records": 1}
 
         # NOTE: Raise because invalid return type.
         with pytest.raises(StageException):
@@ -132,6 +141,11 @@ def test_stage_exec_call(test_path):
             stage: Stage = workflow.job("first-job").stage("call-not-register")
             stage.handler_execute({})
 
+        stage: Stage = workflow.job("second-job").stage("return-model")
+        rs: Result = stage.handler_execute({})
+        assert rs.status == 0
+        assert rs.context == {"name": "foo", "data": {"key": "value"}}
+
 
 @mock.patch.object(Config, "stage_raise_error", True)
 def test_stage_exec_py_raise():

ddeutil_workflow-0.0.46/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.46"