ddeutil-workflow 0.0.26.post1__py3-none-any.whl → 0.0.28__py3-none-any.whl

ddeutil/workflow/utils.py

@@ -5,40 +5,25 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import inspect
 import logging
 import stat
 import time
-from ast import Call, Constant, Expr, Module, Name, parse
 from collections.abc import Iterator
 from datetime import datetime, timedelta
-from functools import wraps
 from hashlib import md5
-from importlib import import_module
 from inspect import isfunction
 from itertools import chain, islice, product
 from pathlib import Path
 from random import randrange
-from typing import Any, Callable, Protocol, TypeVar, Union
+from typing import Any, TypeVar
 from zoneinfo import ZoneInfo
 
-try:
-    from typing import ParamSpec
-except ImportError:
-    from typing_extensions import ParamSpec
+from ddeutil.core import hash_str
 
-from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy
-from ddeutil.io import search_env_replace
-from pydantic import BaseModel
-
-from .__types import DictData, Matrix, Re
+from .__types import DictData, Matrix
 from .conf import config
-from .exceptions import UtilException
 
 T = TypeVar("T")
-P = ParamSpec("P")
-AnyModel = TypeVar("AnyModel", bound=BaseModel)
-AnyModelType = type[AnyModel]
 
 logger = logging.getLogger("ddeutil.workflow")
 
@@ -95,7 +80,7 @@ def gen_id(
     sensitive: bool = True,
     unique: bool = False,
 ) -> str:
-    """Generate running ID for able to tracking. This generate process use `md5`
+    """Generate running ID for able to tracking. This generates process use `md5`
     algorithm function if ``WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE`` set to
     false. But it will cut this hashing value length to 10 it the setting value
     set to true.
@@ -104,6 +89,7 @@ def gen_id(
     :param sensitive: A flag that convert the value to lower case before hashing
     :param unique: A flag that add timestamp at microsecond level to value
         before hashing.
+
     :rtype: str
     """
     if not isinstance(value, str):
@@ -121,84 +107,6 @@ def gen_id(
     ).hexdigest()
 
 
-class TagFunc(Protocol):
-    """Tag Function Protocol"""
-
-    name: str
-    tag: str
-
-    def __call__(self, *args, **kwargs): ...  # pragma: no cov
-
-
-ReturnTagFunc = Callable[P, TagFunc]
-DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
-
-
-def tag(
-    name: str, alias: str | None = None
-) -> DecoratorTagFunc:  # pragma: no cov
-    """Tag decorator function that set function attributes, ``tag`` and ``name``
-    for making registries variable.
-
-    :param: name: A tag name for make different use-case of a function.
-    :param: alias: A alias function name that keeping in registries. If this
-        value does not supply, it will use original function name from __name__.
-    :rtype: Callable[P, TagFunc]
-    """
-
-    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
-        func.tag = name
-        func.name = alias or func.__name__.replace("_", "-")
-
-        @wraps(func)
-        def wrapped(*args, **kwargs):
-            # NOTE: Able to do anything before calling hook function.
-            return func(*args, **kwargs)
-
-        return wrapped
-
-    return func_internal
-
-
-Registry = dict[str, Callable[[], TagFunc]]
-
-
-def make_registry(submodule: str) -> dict[str, Registry]:
-    """Return registries of all functions that able to called with task.
-
-    :param submodule: A module prefix that want to import registry.
-    :rtype: dict[str, Registry]
-    """
-    rs: dict[str, Registry] = {}
-    for module in config.regis_hook:
-        # NOTE: try to sequential import task functions
-        try:
-            importer = import_module(f"{module}.{submodule}")
-        except ModuleNotFoundError:
-            continue
-
-        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
-            # NOTE: check function attribute that already set tag by
-            #   ``utils.tag`` decorator.
-            if not hasattr(func, "tag"):
-                continue
-
-            # NOTE: Create new register name if it not exists
-            if func.name not in rs:
-                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
-                continue
-
-            if func.tag in rs[func.name]:
-                raise ValueError(
-                    f"The tag {func.tag!r} already exists on "
-                    f"{module}.{submodule}, you should change this tag name or "
-                    f"change it func name."
-                )
-            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
-
-    return rs
-
-
 def make_exec(path: str | Path) -> None:
     """Change mode of file to be executable file.
 
@@ -208,314 +116,13 @@ def make_exec(path: str | Path) -> None:
     f.chmod(f.stat().st_mode | stat.S_IEXEC)
 
 
-FILTERS: dict[str, callable] = {  # pragma: no cov
-    "abs": abs,
-    "str": str,
-    "int": int,
-    "title": lambda x: x.title(),
-    "upper": lambda x: x.upper(),
-    "lower": lambda x: x.lower(),
-    "rstr": [str, repr],
-}
-
-
-class FilterFunc(Protocol):
-    """Tag Function Protocol. This protocol that use to represent any callable
-    object that able to access the name attribute.
-    """
-
-    name: str
-
-    def __call__(self, *args, **kwargs): ...  # pragma: no cov
-
-
-def custom_filter(name: str) -> Callable[P, FilterFunc]:
-    """Custom filter decorator function that set function attributes, ``filter``
-    for making filter registries variable.
-
-    :param: name: A filter name for make different use-case of a function.
-    :rtype: Callable[P, FilterFunc]
-    """
-
-    def func_internal(func: Callable[[...], Any]) -> FilterFunc:
-        func.filter = name
-
-        @wraps(func)
-        def wrapped(*args, **kwargs):
-            # NOTE: Able to do anything before calling custom filter function.
-            return func(*args, **kwargs)
-
-        return wrapped
-
-    return func_internal
-
-
-FilterRegistry = Union[FilterFunc, Callable[[...], Any]]
-
-
-def make_filter_registry() -> dict[str, FilterRegistry]:
-    """Return registries of all functions that able to called with task.
-
-    :rtype: dict[str, Registry]
-    """
-    rs: dict[str, Registry] = {}
-    for module in config.regis_filter:
-        # NOTE: try to sequential import task functions
-        try:
-            importer = import_module(module)
-        except ModuleNotFoundError:
-            continue
-
-        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
-            # NOTE: check function attribute that already set tag by
-            #   ``utils.tag`` decorator.
-            if not hasattr(func, "filter"):
-                continue
-
-            rs[func.filter] = import_string(f"{module}.{fstr}")
-
-    rs.update(FILTERS)
-    return rs
-
-
-def get_args_const(
-    expr: str,
-) -> tuple[str, list[Constant], dict[str, Constant]]:
-    """Get arguments and keyword-arguments from function calling string.
-
-    :rtype: tuple[str, list[Constant], dict[str, Constant]]
-    """
-    try:
-        mod: Module = parse(expr)
-    except SyntaxError:
-        raise UtilException(
-            f"Post-filter: {expr} does not valid because it raise syntax error."
-        ) from None
-
-    body: list[Expr] = mod.body
-    if len(body) > 1:
-        raise UtilException(
-            "Post-filter function should be only one calling per workflow."
-        )
-
-    caller: Union[Name, Call]
-    if isinstance((caller := body[0].value), Name):
-        return caller.id, [], {}
-    elif not isinstance(caller, Call):
-        raise UtilException(
-            f"Get arguments does not support for caller type: {type(caller)}"
-        )
-
-    name: Name = caller.func
-    args: list[Constant] = caller.args
-    keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
-
-    if any(not isinstance(i, Constant) for i in args):
-        raise UtilException(f"Argument of {expr} should be constant.")
-
-    if any(not isinstance(i, Constant) for i in keywords.values()):
-        raise UtilException(f"Keyword argument of {expr} should be constant.")
-
-    return name.id, args, keywords
-
-
-def get_args_from_filter(
-    ft: str,
-    filters: dict[str, FilterRegistry],
-) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]:  # pragma: no cov
-    """Get arguments and keyword-arguments from filter function calling string.
-    and validate it with the filter functions mapping dict.
-    """
-    func_name, _args, _kwargs = get_args_const(ft)
-    args: list[Any] = [arg.value for arg in _args]
-    kwargs: dict[Any, Any] = {k: v.value for k, v in _kwargs.items()}
-
-    if func_name not in filters:
-        raise UtilException(
-            f"The post-filter: {func_name!r} does not support yet."
-        )
-
-    if isinstance((f_func := filters[func_name]), list) and (args or kwargs):
-        raise UtilException(
-            "Chain filter function does not support for passing arguments."
-        )
-
-    return func_name, f_func, args, kwargs
-
-
-@custom_filter("fmt")  # pragma: no cov
-def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
-    """Format datetime object to string with the format.
-
-    :param value: A datetime value that want to format to string value.
-    :param fmt: A format string pattern that passing to the `dt.strftime`
-        method.
-
-    :rtype: str
-    """
-    if isinstance(value, datetime):
-        return value.strftime(fmt)
-    raise UtilException(
-        "This custom function should pass input value with datetime type."
-    )
-
-
-def map_post_filter(
-    value: T,
-    post_filter: list[str],
-    filters: dict[str, FilterRegistry],
-) -> T:
-    """Mapping post-filter to value with sequence list of filter function name
-    that will get from the filter registry.
-
-    :param value: A string value that want to mapped with filter function.
-    :param post_filter: A list of post-filter function name.
-    :param filters: A filter registry.
-
-    :rtype: T
-    """
-    for ft in post_filter:
-        func_name, f_func, args, kwargs = get_args_from_filter(ft, filters)
-        try:
-            if isinstance(f_func, list):
-                for func in f_func:
-                    value: T = func(value)
-            else:
-                value: T = f_func(value, *args, **kwargs)
-        except UtilException as err:
-            logger.warning(str(err))
-            raise
-        except Exception as err:
-            logger.warning(str(err))
-            raise UtilException(
-                f"The post-filter function: {func_name} does not fit with "
-                f"{value} (type: {type(value).__name__})."
-            ) from None
-    return value
-
-
-def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
-    """Check value should not pass template with not_in value prefix.
-
-    :param value: A value that want to find parameter template prefix.
-    :param not_in: The not in string that use in the `.startswith` function.
-    :rtype: bool
-    """
-    if isinstance(value, dict):
-        return any(not_in_template(value[k], not_in=not_in) for k in value)
-    elif isinstance(value, (list, tuple, set)):
-        return any(not_in_template(i, not_in=not_in) for i in value)
-    elif not isinstance(value, str):
-        return False
-    return any(
-        (not found.caller.strip().startswith(not_in))
-        for found in Re.finditer_caller(value.strip())
-    )
-
-
-def has_template(value: Any) -> bool:
-    """Check value include templating string.
-
-    :param value: A value that want to find parameter template.
-    :rtype: bool
-    """
-    if isinstance(value, dict):
-        return any(has_template(value[k]) for k in value)
-    elif isinstance(value, (list, tuple, set)):
-        return any(has_template(i) for i in value)
-    elif not isinstance(value, str):
-        return False
-    return bool(Re.RE_CALLER.findall(value.strip()))
-
-
-def str2template(
-    value: str,
-    params: DictData,
-    *,
-    filters: dict[str, FilterRegistry] | None = None,
-) -> Any:
-    """(Sub-function) Pass param to template string that can search by
-    ``RE_CALLER`` regular expression.
-
-        The getter value that map a template should have typing support align
-    with the workflow parameter types that is `str`, `int`, `datetime`, and
-    `list`.
-
-    :param value: A string value that want to mapped with an params
-    :param params: A parameter value that getting with matched regular
-        expression.
-    :param filters:
-    """
-    filters: dict[str, FilterRegistry] = filters or make_filter_registry()
-
-    # NOTE: remove space before and after this string value.
-    value: str = value.strip()
-    for found in Re.finditer_caller(value):
-        # NOTE:
-        #   Get caller and filter values that setting inside;
-        #
-        #   ... ``${{ <caller-value> [ | <filter-value>] ... }}``
-        #
-        caller: str = found.caller
-        pfilter: list[str] = [
-            i.strip()
-            for i in (found.post_filters.strip().removeprefix("|").split("|"))
-            if i != ""
-        ]
-        if not hasdot(caller, params):
-            raise UtilException(f"The params does not set caller: {caller!r}.")
-
-        # NOTE: from validate step, it guarantee that caller exists in params.
-        getter: Any = getdot(caller, params)
-
-        # NOTE:
-        #   If type of getter caller is not string type and it does not use to
-        #   concat other string value, it will return origin value from the
-        #   ``getdot`` function.
-        if value.replace(found.full, "", 1) == "":
-            return map_post_filter(getter, pfilter, filters=filters)
-
-        # NOTE: map post-filter function.
-        getter: Any = map_post_filter(getter, pfilter, filters=filters)
-        if not isinstance(getter, str):
-            getter: str = str(getter)
-
-        value: str = value.replace(found.full, getter, 1)
-
-    return search_env_replace(value)
-
-
-def param2template(
-    value: Any,
-    params: DictData,
-) -> Any:
-    """Pass param to template string that can search by ``RE_CALLER`` regular
-    expression.
-
-    :param value: A value that want to mapped with an params
-    :param params: A parameter value that getting with matched regular
-        expression.
-
-    :rtype: Any
-    :returns: An any getter value from the params input.
-    """
-    filters: dict[str, FilterRegistry] = make_filter_registry()
-    if isinstance(value, dict):
-        return {k: param2template(value[k], params) for k in value}
-    elif isinstance(value, (list, tuple, set)):
-        return type(value)([param2template(i, params) for i in value])
-    elif not isinstance(value, str):
-        return value
-    return str2template(value, params, filters=filters)
-
-
-def filter_func(value: Any) -> Any:
+def filter_func(value: T) -> T:
     """Filter out an own created function of any value of mapping context by
     replacing it to its function name. If it is built-in function, it does not
     have any changing.
 
     :param value: A value context data that want to filter out function value.
-    :type: The same type of an input ``value``.
+    :type: The same type of input ``value``.
     """
     if isinstance(value, dict):
         return {k: filter_func(value[k]) for k in value}
@@ -523,8 +130,8 @@ def filter_func(value: Any) -> Any:
         return type(value)([filter_func(i) for i in value])
 
     if isfunction(value):
-        # NOTE: If it want to improve to get this function, it able to save to
-        #   some global memory storage.
+        # NOTE: If it wants to improve to get this function, it is able to save
+        # to some global memory storage.
         #   ---
         #   >>> GLOBAL_DICT[value.__name__] = value
         #
@@ -540,6 +147,10 @@ def dash2underscore(
 ) -> DictData:
     """Change key name that has dash to underscore.
 
+    :param key
+    :param values
+    :param fixed
+
     :rtype: DictData
     """
     if key in values:
@@ -550,6 +161,8 @@ def dash2underscore(
 def cross_product(matrix: Matrix) -> Iterator[DictData]:
     """Iterator of products value from matrix.
 
+    :param matrix:
+
     :rtype: Iterator[DictData]
     """
     yield from (
@@ -569,6 +182,11 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         ['A', 'B', 'C']
         ['D', 'E', 'F']
         ['G']
+
+    :param iterable:
+    :param n:
+
+    :rtype: Iterator[Any]
     """
     if n < 1:
         raise ValueError("n must be at least one")
@@ -583,7 +201,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
         yield chain((first_el,), chunk_it)
 
 
-def cut_id(run_id: str, *, num: int = 6):
+def cut_id(run_id: str, *, num: int = 6) -> str:
     """Cutting running ID with length.
 
     Example:
@@ -592,6 +210,7 @@ def cut_id(run_id: str, *, num: int = 6):
 
     :param run_id:
     :param num:
-    :return:
+
+    :rtype: str
     """
     return run_id[-num:]

ddeutil/workflow/workflow.py

@@ -48,12 +48,11 @@ from .exceptions import JobException, WorkflowException
 from .job import Job
 from .params import Param
 from .result import Result
+from .templates import has_template, param2template
 from .utils import (
     cut_id,
     gen_id,
     get_dt_now,
-    has_template,
-    param2template,
     wait_a_minute,
 )
 
@@ -139,7 +138,7 @@ class WorkflowQueue:
         """Construct WorkflowQueue object from an input queue value that passing
         with list of datetime or list of WorkflowRelease.
 
-        :raise TypeError: If the type of an input queue does not valid.
+        :raise TypeError: If the type of input queue does not valid.
 
         :rtype: Self
         """
@@ -227,9 +226,9 @@ class WorkflowQueue:
 class Workflow(BaseModel):
     """Workflow Pydantic model.
 
-        This is the main future of this project because it use to be workflow
+        This is the main future of this project because it uses to be workflow
     data for running everywhere that you want or using it to scheduler task in
-    background. It use lightweight coding line from Pydantic Model and enhance
+    background. It uses lightweight coding line from Pydantic Model and enhance
     execute method on it.
     """
 
@@ -318,7 +317,7 @@ class Workflow(BaseModel):
     @model_validator(mode="before")
     def __prepare_model_before__(cls, values: DictData) -> DictData:
         """Prepare the params key in the data model before validating."""
-        # NOTE: Prepare params type if it passing with only type value.
+        # NOTE: Prepare params type if it is passing with only type value.
         if params := values.pop("params", {}):
             values["params"] = {
                 p: (
@@ -342,7 +341,7 @@ class Workflow(BaseModel):
     @field_validator("on", mode="after")
     def __on_no_dup_and_reach_limit__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
-        contain the every minute value more than one value, it will remove to
+        contains the every minute value more than one value, it will remove to
         only one value.
 
         :raise ValueError: If it has some duplicate value.
@@ -360,8 +359,8 @@ class Workflow(BaseModel):
         # WARNING:
         # if '* * * * *' in set_ons and len(set_ons) > 1:
         #     raise ValueError(
-        #         "If it has every minute cronjob on value, it should has only "
-        #         "one value in the on field."
+        #         "If it has every minute cronjob on value, it should have "
+        #         "only one value in the on field."
         #     )
 
         if len(set_ons) > config.max_on_per_workflow:
@@ -373,7 +372,7 @@ class Workflow(BaseModel):
 
     @model_validator(mode="after")
     def __validate_jobs_need__(self) -> Self:
-        """Validate each need job in any jobs should exists.
+        """Validate each need job in any jobs should exist.
 
         :raise WorkflowException: If it has not exists need value in this
             workflow job.
@@ -486,7 +485,7 @@ class Workflow(BaseModel):
             - Initialize WorkflowQueue and WorkflowRelease if they do not pass.
             - Create release data for pass to parameter templating function.
             - Execute this workflow with mapping release data to its parameters.
-            - Writing log
+            - Writing result log
             - Remove this release on the running queue
             - Push this release to complete queue
 
@@ -592,10 +591,10 @@ class Workflow(BaseModel):
         """Generate queue of datetime from the cron runner that initialize from
         the on field. with offset value.
 
-        :param offset: A offset in second unit for time travel.
+        :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to making log object.
+        :param log: A log class that want to make log object.
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.
 
@@ -697,7 +696,7 @@ class Workflow(BaseModel):
             start_date: datetime = current_date
             offset: float = 0
 
-        # NOTE: End date is use to stop generate queue with an input periods
+        # NOTE: End date is using to stop generate queue with an input periods
         #   value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
@@ -813,7 +812,7 @@ class Workflow(BaseModel):
         :param params: A params that was parameterized from workflow execution.
         :param run_id: A workflow running ID for this job execution.
         :param raise_error: A flag that raise error instead catching to result
-            if it get exception from job execution.
+            if it gets exception from job execution.
 
         :rtype: Result
         :return: Return the result object that receive the job execution result
@@ -869,8 +868,8 @@ class Workflow(BaseModel):
         """Execute workflow with passing a dynamic parameters to all jobs that
         included in this workflow model with ``jobs`` field.
 
-            The result of execution process for each jobs and stages on this
-        workflow will keeping in dict which able to catch out with all jobs and
+            The result of execution process for each job and stages on this
+        workflow will keep in dict which able to catch out with all jobs and
         stages by dot annotation.
 
             For example, when I want to use the output from previous stage, I
@@ -908,8 +907,8 @@ class Workflow(BaseModel):
             )
             return rs.catch(status=0, context=params)
 
-        # NOTE: Create a job queue that keep the job that want to running after
-        #   it dependency condition.
+        # NOTE: Create a job queue that keep the job that want to run after
+        #   its dependency condition.
         jq: Queue = Queue()
         for job_id in self.jobs:
             jq.put(job_id)
@@ -968,7 +967,7 @@ class Workflow(BaseModel):
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
-            timeout.
+            time out.
         :param job_queue: A job queue object.
         :param timeout: A second value unit that bounding running time.
         :param thread_timeout: A timeout to waiting all futures complete.
@@ -1065,7 +1064,7 @@ class Workflow(BaseModel):
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
-            timeout.
+            time out.
         :param timeout: A second value unit that bounding running time.
 
         :rtype: DictData
@@ -1091,7 +1090,7 @@ class Workflow(BaseModel):
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
-            #   before release. This job execution process will running until
+            #   before release. This job execution process will run until
             #   done before checking all execution timeout or not.
             #
             #   {
@@ -1183,7 +1182,7 @@ class WorkflowTask:
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to making log object.
+        :param log: A log class that want to make log object.
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.