ddeutil-workflow 0.0.39__py3-none-any.whl → 0.0.41__py3-none-any.whl

ddeutil/workflow/{templates.py → reusables.py}

@@ -3,22 +3,25 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use config
+"""Reusables module that keep any templating functions."""
 from __future__ import annotations
 
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
+from dataclasses import dataclass
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
-from typing import Any, Callable, Protocol, TypeVar, Union
+from typing import Any, Callable, Optional, Protocol, TypeVar, Union
 
 try:
     from typing import ParamSpec
 except ImportError:
     from typing_extensions import ParamSpec
 
-from ddeutil.core import getdot, import_string
+from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
 
 from .__types import DictData, Re
@@ -29,6 +32,7 @@ T = TypeVar("T")
 P = ParamSpec("P")
 
 logger = logging.getLogger("ddeutil.workflow")
+logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
 FILTERS: dict[str, callable] = {  # pragma: no cov
@@ -77,13 +81,17 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
     return func_internal
 
 
-def make_filter_registry() -> dict[str, FilterRegistry]:
+def make_filter_registry(
+    registers: Optional[list[str]] = None,
+) -> dict[str, FilterRegistry]:
     """Return registries of all functions that able to called with task.
 
+    :param registers: (Optional[list[str]]) Override list of register.
+
     :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
-    for module in config.regis_filter:
+    for module in registers or config.regis_filter:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -251,6 +259,7 @@ def str2template(
     params: DictData,
     *,
     filters: dict[str, FilterRegistry] | None = None,
+    registers: Optional[list[str]] = None,
 ) -> str:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
@@ -263,10 +272,13 @@ def str2template(
     :param params: (DictData) A parameter value that getting with matched
         regular expression.
     :param filters: A mapping of filter registry.
+    :param registers: (Optional[list[str]]) Override list of register.
 
     :rtype: str
     """
-    filters: dict[str, FilterRegistry] = filters or make_filter_registry()
+    filters: dict[str, FilterRegistry] = filters or make_filter_registry(
+        registers
+    )
 
     # NOTE: remove space before and after this string value.
     value: str = value.strip()
@@ -315,6 +327,8 @@ def param2template(
     value: T,
     params: DictData,
     filters: dict[str, FilterRegistry] | None = None,
+    *,
+    registers: Optional[list[str]] = None,
 ) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
@@ -323,18 +337,29 @@ def param2template(
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters: A filter mapping for mapping with `map_post_filter` func.
+    :param registers: (Optional[list[str]]) Override list of register.
 
     :rtype: T
     :returns: An any getter value from the params input.
     """
-    filters: dict[str, FilterRegistry] = filters or make_filter_registry()
+    filters: dict[str, FilterRegistry] = filters or make_filter_registry(
+        registers
+    )
     if isinstance(value, dict):
-        return {k: param2template(value[k], params, filters) for k in value}
+        return {
+            k: param2template(value[k], params, filters, registers=registers)
+            for k in value
+        }
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([param2template(i, params, filters) for i in value])
+        return type(value)(
+            [
+                param2template(i, params, filters, registers=registers)
+                for i in value
+            ]
+        )
     elif not isinstance(value, str):
         return value
-    return str2template(value, params, filters=filters)
+    return str2template(value, params, filters=filters, registers=registers)
 
 
 @custom_filter("fmt")  # pragma: no cov
@@ -359,3 +384,164 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 def coalesce(value: T | None, default: Any) -> T:
     """Coalesce with default value if the main value is None."""
     return default if value is None else value
+
+
+class TagFunc(Protocol):
+    """Tag Function Protocol"""
+
+    name: str
+    tag: str
+
+    def __call__(self, *args, **kwargs): ...  # pragma: no cov
+
+
+ReturnTagFunc = Callable[P, TagFunc]
+DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
+
+
+def tag(
+    name: str, alias: str | None = None
+) -> DecoratorTagFunc:  # pragma: no cov
+    """Tag decorator function that set function attributes, ``tag`` and ``name``
+    for making registries variable.
+
+    :param: name: (str) A tag name for make different use-case of a function.
+    :param: alias: (str) A alias function name that keeping in registries.
+        If this value does not supply, it will use original function name
+        from `__name__` argument.
+
+    :rtype: Callable[P, TagFunc]
+    """
+
+    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
+        func.tag = name
+        func.name = alias or func.__name__.replace("_", "-")
+
+        @wraps(func)
+        def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            """Wrapped function."""
+            return func(*args, **kwargs)
+
+        @wraps(func)
+        async def async_wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            """Wrapped async function."""
+            return await func(*args, **kwargs)
+
+        return async_wrapped if inspect.iscoroutinefunction(func) else wrapped
+
+    return func_internal
+
+
+Registry = dict[str, Callable[[], TagFunc]]
+
+
+def make_registry(
+    submodule: str,
+    registries: Optional[list[str]] = None,
+) -> dict[str, Registry]:
+    """Return registries of all functions that able to called with task.
+
+    :param submodule: (str) A module prefix that want to import registry.
+    :param registries: (Optional[list[str]]) A list of registry.
+
+    :rtype: dict[str, Registry]
+    """
+    rs: dict[str, Registry] = {}
+    regis_calls: list[str] = registries or config.regis_call  # pragma: no cov
+    regis_calls.extend(["ddeutil.vendors"])
+    for module in regis_calls:
+        # NOTE: try to sequential import task functions
+        try:
+            importer = import_module(f"{module}.{submodule}")
+        except ModuleNotFoundError:
+            continue
+
+        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
+            # NOTE: check function attribute that already set tag by
+            #   ``utils.tag`` decorator.
+            if not (
+                hasattr(func, "tag") and hasattr(func, "name")
+            ):  # pragma: no cov
+                continue
+
+            # NOTE: Define type of the func value.
+            func: TagFunc
+
+            # NOTE: Create new register name if it not exists
+            if func.name not in rs:
+                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
+                continue
+
+            if func.tag in rs[func.name]:
+                raise ValueError(
+                    f"The tag {func.tag!r} already exists on "
+                    f"{module}.{submodule}, you should change this tag name or "
+                    f"change it func name."
+                )
+            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
+
+    return rs
+
+
+@dataclass(frozen=True)
+class CallSearchData:
+    """Call Search dataclass that use for receive regular expression grouping
+    dict from searching call string value.
+    """
+
+    path: str
+    func: str
+    tag: str
+
+
+def extract_call(
+    call: str,
+    registries: Optional[list[str]] = None,
+) -> Callable[[], TagFunc]:
+    """Extract Call function from string value to call partial function that
+    does run it at runtime.
+
+    :param call: (str) A call value that able to match with Task regex.
+    :param registries: (Optional[list[str]]) A list of registry.
+
+        The format of call value should contain 3 regular expression groups
+    which match with the below config format:
+
+        >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
+
+    Examples:
+        >>> extract_call("tasks/el-postgres-to-delta@polars")
+        ...
+        >>> extract_call("tasks/return-type-not-valid@raise")
+        ...
+
+    :raise NotImplementedError: When the searching call's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching call's tag result does not
+        exist in the registry with its function key.
+
+    :rtype: Callable[[], TagFunc]
+    """
+    if not (found := Re.RE_TASK_FMT.search(call)):
+        raise ValueError(
+            f"Call {call!r} does not match with the call regex format."
+        )
+
+    call: CallSearchData = CallSearchData(**found.groupdict())
+    rgt: dict[str, Registry] = make_registry(
+        submodule=f"{call.path}",
+        registries=registries,
+    )
+
+    if call.func not in rgt:
+        raise NotImplementedError(
+            f"`REGISTER-MODULES.{call.path}.registries` not implement "
+            f"registry: {call.func!r}."
+        )
+
+    if call.tag not in rgt[call.func]:
+        raise NotImplementedError(
+            f"tag: {call.tag!r} not found on registry func: "
+            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
+        )
+    return rgt[call.func][call.tag]

ddeutil/workflow/scheduler.py

@@ -31,6 +31,7 @@ from concurrent.futures import (
 from datetime import datetime, timedelta
 from functools import wraps
 from heapq import heappop, heappush
+from pathlib import Path
 from textwrap import dedent
 from threading import Thread
 from typing import Callable, Optional, TypedDict, Union
@@ -51,10 +52,10 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .audit import Audit, get_audit
-from .conf import Loader, config, get_logger
+from .conf import Loader, SimLoad, config, get_logger
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
+from .logs import Audit, get_audit
 from .result import Result, Status
 from .utils import batch, delay
 from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
@@ -266,6 +267,8 @@ class Schedule(BaseModel):
         :param externals: An external parameters that want to pass to Loader
             object.
 
+        :raise ValueError: If the type does not match with current object.
+
         :rtype: Self
         """
         loader: Loader = Loader(name, externals=(externals or {}))
@@ -281,6 +284,42 @@ class Schedule(BaseModel):
 
         return cls.model_validate(obj=loader_data)
 
+    @classmethod
+    def from_path(
+        cls,
+        name: str,
+        path: Path,
+        externals: DictData | None = None,
+    ) -> Self:
+        """Create Schedule instance from the SimLoad object that receive an
+        input schedule name and conf path. The loader object will use this
+        schedule name to searching configuration data of this schedule model
+        in conf path.
+
+        :param name: (str) A schedule name that want to pass to Loader object.
+        :param path: (Path) A config path that want to search.
+        :param externals: An external parameters that want to pass to Loader
+            object.
+
+        :raise ValueError: If the type does not match with current object.
+
+        :rtype: Self
+        """
+        loader: SimLoad = SimLoad(
+            name, conf_path=path, externals=(externals or {})
+        )
+
+        # NOTE: Validate the config type match with current connection model
+        if loader.type != cls.__name__:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+
+        loader_data: DictData = copy.deepcopy(loader.data)
+
+        # NOTE: Add name to loader data
+        loader_data["name"] = name.replace(" ", "_")
+
+        return cls.model_validate(obj=loader_data)
+
     def tasks(
         self,
         start_date: datetime,