ddeutil-workflow 0.0.40__py3-none-any.whl → 0.0.42__py3-none-any.whl

ddeutil/workflow/{templates.py → reusables.py}

@@ -3,32 +3,36 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use dynamic config
+"""Reusables module that keep any templating functions."""
 from __future__ import annotations
 
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
+from dataclasses import dataclass
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
-from typing import Any, Callable, Protocol, TypeVar, Union
+from typing import Any, Callable, Optional, Protocol, TypeVar, Union
 
 try:
     from typing import ParamSpec
 except ImportError:
     from typing_extensions import ParamSpec
 
-from ddeutil.core import getdot, import_string
+from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
 
 from .__types import DictData, Re
-from .conf import config
+from .conf import dynamic
 from .exceptions import UtilException
 
 T = TypeVar("T")
 P = ParamSpec("P")
 
 logger = logging.getLogger("ddeutil.workflow")
+logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
 FILTERS: dict[str, callable] = {  # pragma: no cov
@@ -77,13 +81,17 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
     return func_internal
 
 
-def make_filter_registry() -> dict[str, FilterRegistry]:
+def make_filter_registry(
+    registers: Optional[list[str]] = None,
+) -> dict[str, FilterRegistry]:
     """Return registries of all functions that able to called with task.
 
+    :param registers: (Optional[list[str]]) Override list of register.
+
     :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
-    for module in config.regis_filter:
+    for module in dynamic("regis_filter", f=registers):
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -251,6 +259,7 @@ def str2template(
     params: DictData,
     *,
     filters: dict[str, FilterRegistry] | None = None,
+    registers: Optional[list[str]] = None,
 ) -> str:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
@@ -263,10 +272,13 @@ def str2template(
     :param params: (DictData) A parameter value that getting with matched
         regular expression.
     :param filters: A mapping of filter registry.
+    :param registers: (Optional[list[str]]) Override list of register.
 
     :rtype: str
     """
-    filters: dict[str, FilterRegistry] = filters or make_filter_registry()
+    filters: dict[str, FilterRegistry] = filters or make_filter_registry(
+        registers=registers
+    )
 
     # NOTE: remove space before and after this string value.
     value: str = value.strip()
@@ -315,6 +327,8 @@ def param2template(
     value: T,
     params: DictData,
     filters: dict[str, FilterRegistry] | None = None,
+    *,
+    extras: Optional[DictData] = None,
 ) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
@@ -323,18 +337,29 @@ def param2template(
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters: A filter mapping for mapping with `map_post_filter` func.
+    :param extras: (Optional[list[str]]) An Override extras.
 
     :rtype: T
     :returns: An any getter value from the params input.
     """
-    filters: dict[str, FilterRegistry] = filters or make_filter_registry()
+    registers: Optional[list[str]] = (
+        extras.get("regis_filter") if extras else None
+    )
+    filters: dict[str, FilterRegistry] = filters or make_filter_registry(
+        registers=registers
+    )
     if isinstance(value, dict):
-        return {k: param2template(value[k], params, filters) for k in value}
+        return {
+            k: param2template(value[k], params, filters, extras=extras)
+            for k in value
+        }
     elif isinstance(value, (list, tuple, set)):
-        return type(value)([param2template(i, params, filters) for i in value])
+        return type(value)(
+            [param2template(i, params, filters, extras=extras) for i in value]
+        )
     elif not isinstance(value, str):
         return value
-    return str2template(value, params, filters=filters)
+    return str2template(value, params, filters=filters, registers=registers)
 
 
 @custom_filter("fmt")  # pragma: no cov
@@ -359,3 +384,167 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 def coalesce(value: T | None, default: Any) -> T:
     """Coalesce with default value if the main value is None."""
     return default if value is None else value
+
+
+class TagFunc(Protocol):
+    """Tag Function Protocol"""
+
+    name: str
+    tag: str
+
+    def __call__(self, *args, **kwargs): ...  # pragma: no cov
+
+
+ReturnTagFunc = Callable[P, TagFunc]
+DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
+
+
+def tag(
+    name: str, alias: str | None = None
+) -> DecoratorTagFunc:  # pragma: no cov
+    """Tag decorator function that set function attributes, ``tag`` and ``name``
+    for making registries variable.
+
+    :param: name: (str) A tag name for make different use-case of a function.
+    :param: alias: (str) A alias function name that keeping in registries.
+        If this value does not supply, it will use original function name
+        from `__name__` argument.
+
+    :rtype: Callable[P, TagFunc]
+    """
+
+    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
+        func.tag = name
+        func.name = alias or func.__name__.replace("_", "-")
+
+        @wraps(func)
+        def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            """Wrapped function."""
+            return func(*args, **kwargs)
+
+        @wraps(func)
+        async def async_wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            """Wrapped async function."""
+            return await func(*args, **kwargs)
+
+        return async_wrapped if inspect.iscoroutinefunction(func) else wrapped
+
+    return func_internal
+
+
+Registry = dict[str, Callable[[], TagFunc]]
+
+
+def make_registry(
+    submodule: str,
+    registries: Optional[list[str]] = None,
+) -> dict[str, Registry]:
+    """Return registries of all functions that able to called with task.
+
+    :param submodule: (str) A module prefix that want to import registry.
+    :param registries: (Optional[list[str]]) A list of registry.
+
+    :rtype: dict[str, Registry]
+    """
+    rs: dict[str, Registry] = {}
+    regis_calls: list[str] = dynamic(
+        "regis_call", f=registries
+    )  # pragma: no cov
+    regis_calls.extend(["ddeutil.vendors"])
+
+    for module in regis_calls:
+        # NOTE: try to sequential import task functions
+        try:
+            importer = import_module(f"{module}.{submodule}")
+        except ModuleNotFoundError:
+            continue
+
+        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
+            # NOTE: check function attribute that already set tag by
+            #   ``utils.tag`` decorator.
+            if not (
+                hasattr(func, "tag") and hasattr(func, "name")
+            ):  # pragma: no cov
+                continue
+
+            # NOTE: Define type of the func value.
+            func: TagFunc
+
+            # NOTE: Create new register name if it not exists
+            if func.name not in rs:
+                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
+                continue
+
+            if func.tag in rs[func.name]:
+                raise ValueError(
+                    f"The tag {func.tag!r} already exists on "
+                    f"{module}.{submodule}, you should change this tag name or "
+                    f"change it func name."
+                )
+            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
+
+    return rs
+
+
+@dataclass(frozen=True)
+class CallSearchData:
+    """Call Search dataclass that use for receive regular expression grouping
+    dict from searching call string value.
+    """
+
+    path: str
+    func: str
+    tag: str
+
+
+def extract_call(
+    call: str,
+    registries: Optional[list[str]] = None,
+) -> Callable[[], TagFunc]:
+    """Extract Call function from string value to call partial function that
+    does run it at runtime.
+
+    :param call: (str) A call value that able to match with Task regex.
+    :param registries: (Optional[list[str]]) A list of registry.
+
+        The format of call value should contain 3 regular expression groups
+    which match with the below config format:
+
+        >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
+
+    Examples:
+        >>> extract_call("tasks/el-postgres-to-delta@polars")
+        ...
+        >>> extract_call("tasks/return-type-not-valid@raise")
+        ...
+
+    :raise NotImplementedError: When the searching call's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching call's tag result does not
+        exist in the registry with its function key.
+
+    :rtype: Callable[[], TagFunc]
+    """
+    if not (found := Re.RE_TASK_FMT.search(call)):
+        raise ValueError(
+            f"Call {call!r} does not match with the call regex format."
+        )
+
+    call: CallSearchData = CallSearchData(**found.groupdict())
+    rgt: dict[str, Registry] = make_registry(
+        submodule=f"{call.path}",
+        registries=registries,
+    )
+
+    if call.func not in rgt:
+        raise NotImplementedError(
+            f"`REGISTER-MODULES.{call.path}.registries` not implement "
+            f"registry: {call.func!r}."
+        )
+
+    if call.tag not in rgt[call.func]:
+        raise NotImplementedError(
+            f"tag: {call.tag!r} not found on registry func: "
+            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
+        )
+    return rgt[call.func][call.tag]

ddeutil/workflow/scheduler.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""
-The main schedule running is `schedule_runner` function that trigger the
+# [x] Use fix config
+"""The main schedule running is `schedule_runner` function that trigger the
 multiprocess of `schedule_control` function for listing schedules on the
 config by `Loader.finds(Schedule)`.
 
@@ -52,10 +52,10 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .audit import Audit, get_audit
 from .conf import Loader, SimLoad, config, get_logger
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
+from .logs import Audit, get_audit
 from .result import Result, Status
 from .utils import batch, delay
 from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
@@ -86,9 +86,9 @@ class ScheduleWorkflow(BaseModel):
     model.
 
         This on field does not equal to the on field of Workflow model, but it
-    uses same logic to generate running release date with crontab object. It use
-    for override the on field if the schedule time was change but you do not
-    want to change on the workflow model.
+    uses same logic to generate running release date with crontab object. It
+    uses for override the on field if the schedule time was change, but you do
+    not want to change on the workflow model.
     """
 
     alias: Optional[str] = Field(
@@ -177,7 +177,7 @@ class ScheduleWorkflow(BaseModel):
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
         *,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field.
@@ -187,17 +187,17 @@ class ScheduleWorkflow(BaseModel):
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
-        :param externals: An external parameters that pass to the Loader object.
+        :param extras: An extra parameters that pass to the Loader object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
         workflow_tasks: list[WorkflowTask] = []
-        extras: DictData = externals or {}
+        extras: DictData = extras or {}
 
         # NOTE: Loading workflow model from the name of workflow.
-        wf: Workflow = Workflow.from_loader(self.name, externals=extras)
+        wf: Workflow = Workflow.from_conf(self.name, extras=extras)
         wf_queue: ReleaseQueue = queue[self.alias]
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
@@ -254,24 +254,24 @@ class Schedule(BaseModel):
         return dedent(value)
 
     @classmethod
-    def from_loader(
+    def from_conf(
         cls,
         name: str,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> Self:
         """Create Schedule instance from the Loader object that only receive
         an input schedule name. The loader object will use this schedule name to
         searching configuration data of this schedule model in conf path.
 
         :param name: (str) A schedule name that want to pass to Loader object.
-        :param externals: An external parameters that want to pass to Loader
+        :param extras: An extra parameters that want to pass to Loader
             object.
 
         :raise ValueError: If the type does not match with current object.
 
         :rtype: Self
         """
-        loader: Loader = Loader(name, externals=(externals or {}))
+        loader: Loader = Loader(name, externals=(extras or {}))
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -325,16 +325,16 @@ class Schedule(BaseModel):
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
         *,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field from workflow schedule model.
 
         :param start_date: A start date that get from the workflow schedule.
-        :param queue: A mapping of name and list of datetime for queue.
-        :type queue: dict[str, ReleaseQueue]
-        :param externals: An external parameters that pass to the Loader object.
-        :type externals: DictData | None
+        :param queue: (dict[str, ReleaseQueue]) A mapping of name and list of
+            datetime for queue.
+        :param extras: (DictData) An extra parameters that pass to the Loader
+            object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
@@ -348,7 +348,7 @@ class Schedule(BaseModel):
                 queue[workflow.alias] = ReleaseQueue()
 
             workflow_tasks.extend(
-                workflow.tasks(start_date, queue=queue, externals=externals)
+                workflow.tasks(start_date, queue=queue, extras=extras)
             )
 
         return workflow_tasks
@@ -357,14 +357,14 @@ class Schedule(BaseModel):
         self,
         *,
         stop: datetime | None = None,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
         audit: type[Audit] | None = None,
         parent_run_id: str | None = None,
     ) -> Result:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
         :param stop: A datetime value that use to stop running schedule.
-        :param externals: An external parameters that pass to Loader.
+        :param extras: An extra parameters that pass to Loader.
         :param audit: An audit class that use on the workflow task release for
             writing its release audit context.
         :param parent_run_id: A parent workflow running ID for this release.
@@ -385,9 +385,7 @@ class Schedule(BaseModel):
         ) + timedelta(minutes=1)
 
         scheduler_pending(
-            tasks=self.tasks(
-                start_date_waiting, queue=queue, externals=externals
-            ),
+            tasks=self.tasks(start_date_waiting, queue=queue, extras=extras),
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -709,7 +707,7 @@ def scheduler_pending(
 def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
-    externals: DictData | None = None,
+    extras: DictData | None = None,
     *,
     audit: type[Audit] | None = None,
     parent_run_id: str | None = None,
@@ -720,7 +718,7 @@ def schedule_control(
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: A datetime value that use to stop running schedule.
-    :param externals: An external parameters that pass to Loader.
+    :param extras: An extra parameters that pass to Loader.
     :param audit: An audit class that use on the workflow task release for
         writing its release audit context.
     :param parent_run_id: A parent workflow running ID for this release.
@@ -745,10 +743,10 @@ def schedule_control(
     tasks: list[WorkflowTask] = []
     for name in schedules:
         tasks.extend(
-            Schedule.from_loader(name, externals=externals).tasks(
+            Schedule.from_conf(name, extras=extras).tasks(
                 start_date_waiting,
                 queue=queue,
-                externals=externals,
+                extras=extras,
             ),
         )