ddeutil-workflow 0.0.15__py3-none-any.whl → 0.0.17__py3-none-any.whl

ddeutil/workflow/utils.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 
 import inspect
 import logging
-import os
 import stat
 import time
 from abc import ABC, abstractmethod
@@ -15,7 +14,7 @@ from ast import Call, Constant, Expr, Module, Name, parse
 from collections.abc import Iterator
 from dataclasses import field
 from datetime import date, datetime
-from functools import cached_property, wraps
+from functools import wraps
 from hashlib import md5
 from importlib import import_module
 from inspect import isfunction
@@ -30,12 +29,10 @@ try:
 except ImportError:
     from typing_extensions import ParamSpec
 
-from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy, str2bool
-from ddeutil.io import PathData, PathSearch, YamlFlResolve, search_env_replace
-from ddeutil.io.models.lineage import dt_now
-from pydantic import BaseModel, ConfigDict, Field
+from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy
+from ddeutil.io import search_env_replace
+from pydantic import BaseModel, Field
 from pydantic.dataclasses import dataclass
-from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
@@ -43,6 +40,7 @@ from .__types import DictData, Matrix, Re
 from .conf import config
 from .exceptions import ParamValueException, UtilException
 
+T = TypeVar("T")
 P = ParamSpec("P")
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 AnyModelType = type[AnyModel]
@@ -50,16 +48,30 @@ AnyModelType = type[AnyModel]
 logger = logging.getLogger("ddeutil.workflow")
 
 
-def get_diff_sec(dt: datetime, tz: ZoneInfo | None = None) -> int:
+def get_dt_now(tz: ZoneInfo | None = None) -> datetime:  # pragma: no cov
+    """Return the current datetime object.
+
+    :param tz:
+    :return: The current datetime object that use an input timezone or UTC.
+    """
+    return datetime.now(tz=(tz or ZoneInfo("UTC")))
+
+
+def get_diff_sec(
+    dt: datetime, tz: ZoneInfo | None = None
+) -> int:  # pragma: no cov
     """Return second value that come from diff of an input datetime and the
     current datetime with specific timezone.
+
+    :param dt:
+    :param tz:
     """
     return round(
         (dt - datetime.now(tz=(tz or ZoneInfo("UTC")))).total_seconds()
     )
 
 
-def delay(second: float = 0) -> None:
+def delay(second: float = 0) -> None:  # pragma: no cov
     """Delay time that use time.sleep with random second value between
     0.00 - 0.99 seconds.
 
@@ -68,203 +80,6 @@ def delay(second: float = 0) -> None:
     time.sleep(second + randrange(0, 99, step=10) / 100)
 
 
-class Engine(BaseModel):
-    """Engine Model"""
-
-    paths: PathData = Field(default_factory=PathData)
-    registry: list[str] = Field(
-        default_factory=lambda: ["ddeutil.workflow"],  # pragma: no cover
-    )
-    registry_filter: list[str] = Field(
-        default_factory=lambda: ["ddeutil.workflow.utils"],  # pragma: no cover
-    )
-
-    @model_validator(mode="before")
-    def __prepare_registry(cls, values: DictData) -> DictData:
-        """Prepare registry value that passing with string type. It convert the
-        string type to list of string.
-        """
-        if (_regis := values.get("registry")) and isinstance(_regis, str):
-            values["registry"] = [_regis]
-        if (_regis_filter := values.get("registry_filter")) and isinstance(
-            _regis, str
-        ):
-            values["registry_filter"] = [_regis_filter]
-        return values
-
-
-class CoreConf(BaseModel):
-    """Core Config Model"""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    tz: ZoneInfo = Field(default_factory=lambda: ZoneInfo("UTC"))
-
-
-class ConfParams(BaseModel):
-    """Params Model"""
-
-    engine: Engine = Field(
-        default_factory=Engine,
-        description="A engine mapping values.",
-    )
-    core: CoreConf = Field(
-        default_factory=CoreConf,
-        description="A core config value",
-    )
-
-
-def load_config() -> ConfParams:
-    """Load Config data from ``workflows-conf.yaml`` file.
-
-    Configuration Docs:
-    ---
-    :var engine.registry:
-    :var engine.registry_filter:
-    :var paths.root:
-    :var paths.conf:
-    """
-    root_path: str = os.getenv("WORKFLOW_ROOT_PATH", ".")
-
-    regis: list[str] = ["ddeutil.workflow"]
-    if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
-        regis = [r.strip() for r in regis_env.split(",")]
-
-    regis_filter: list[str] = ["ddeutil.workflow.utils"]
-    if regis_filter_env := os.getenv("WORKFLOW_CORE_REGISTRY_FILTER"):
-        regis_filter = [r.strip() for r in regis_filter_env.split(",")]
-
-    conf_path: str = (
-        f"{root_path}/{conf_env}"
-        if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
-        else None
-    )
-    return ConfParams.model_validate(
-        obj={
-            "engine": {
-                "registry": regis,
-                "registry_filter": regis_filter,
-                "paths": {
-                    "root": root_path,
-                    "conf": conf_path,
-                },
-            },
-        }
-    )
-
-
-class SimLoad:
-    """Simple Load Object that will search config data by given some identity
-    value like name of workflow or on.
-
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param params: A Params model object.
-    :param externals: An external parameters
-
-    Noted:
-
-        The config data should have ``type`` key for modeling validation that
-    make this loader know what is config should to do pass to.
-
-        ... <identity-key>:
-        ...     type: <importable-object>
-        ...     <key-data>: <value-data>
-        ...     ...
-
-    """
-
-    def __init__(
-        self,
-        name: str,
-        params: ConfParams,
-        externals: DictData | None = None,
-    ) -> None:
-        self.data: DictData = {}
-        for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
-                data := YamlFlResolve(file).read().get(name, {})
-            ):
-                self.data = data
-
-        # VALIDATE: check the data that reading should not empty.
-        if not self.data:
-            raise ValueError(f"Config {name!r} does not found on conf path")
-
-        self.conf_params: ConfParams = params
-        self.externals: DictData = externals or {}
-        self.data.update(self.externals)
-
-    @classmethod
-    def finds(
-        cls,
-        obj: object,
-        params: ConfParams,
-        *,
-        include: list[str] | None = None,
-        exclude: list[str] | None = None,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Find all data that match with object type in config path. This class
-        method can use include and exclude list of identity name for filter and
-        adds-on.
-
-        :param obj:
-        :param params:
-        :param include:
-        :param exclude:
-        :rtype: Iterator[tuple[str, DictData]]
-        """
-        exclude: list[str] = exclude or []
-        for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
-                values := YamlFlResolve(file).read()
-            ):
-                for key, data in values.items():
-                    if key in exclude:
-                        continue
-                    if issubclass(get_type(data["type"], params), obj) and (
-                        include is None or all(i in data for i in include)
-                    ):
-                        yield key, data
-
-    @cached_property
-    def type(self) -> AnyModelType:
-        """Return object of string type which implement on any registry. The
-        object type.
-
-        :rtype: AnyModelType
-        """
-        if not (_typ := self.data.get("type")):
-            raise ValueError(
-                f"the 'type' value: {_typ} does not exists in config data."
-            )
-        return get_type(_typ, self.conf_params)
-
-
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
-
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param externals: An external parameters
-    """
-
-    @classmethod
-    def finds(
-        cls,
-        obj: object,
-        *,
-        include: list[str] | None = None,
-        exclude: list[str] | None = None,
-        **kwargs,
-    ) -> DictData:
-        """Override the find class method from the Simple Loader object."""
-        return super().finds(
-            obj=obj, params=load_config(), include=include, exclude=exclude
-        )
-
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, load_config(), externals)
-
-
 def gen_id(
     value: Any,
     *,
@@ -272,9 +87,9 @@ def gen_id(
     unique: bool = False,
 ) -> str:
     """Generate running ID for able to tracking. This generate process use `md5`
-    algorithm function if ``WORKFLOW_CORE_PIPELINE_ID_SIMPLE`` set to false.
-    But it will cut this hashing value length to 10 it the setting value set to
-    true.
+    algorithm function if ``WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE`` set to
+    false. But it will cut this hashing value length to 10 it the setting value
+    set to true.
 
     :param value: A value that want to add to prefix before hashing with md5.
     :param sensitive: A flag that convert the value to lower case before hashing
@@ -285,7 +100,7 @@ def gen_id(
     if not isinstance(value, str):
         value: str = str(value)
 
-    if str2bool(os.getenv("WORKFLOW_CORE_PIPELINE_ID_SIMPLE", "true")):
+    if config.workflow_id_simple_mode:
         return hash_str(f"{(value if sensitive else value.lower())}", n=10) + (
             f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else ""
         )
@@ -297,40 +112,22 @@ def gen_id(
     ).hexdigest()
 
 
-def get_type(t: str, params: ConfParams) -> AnyModelType:
-    """Return import type from string importable value in the type key.
-
-    :param t: A importable type string.
-    :param params: A config parameters that use registry to search this
-        type.
-    :rtype: AnyModelType
-    """
-    try:
-        # NOTE: Auto adding module prefix if it does not set
-        return import_string(f"ddeutil.workflow.{t}")
-    except ModuleNotFoundError:
-        for registry in params.engine.registry:
-            try:
-                return import_string(f"{registry}.{t}")
-            except ModuleNotFoundError:
-                continue
-        return import_string(f"{t}")
-
-
 class TagFunc(Protocol):
     """Tag Function Protocol"""
 
     name: str
     tag: str
 
-    def __call__(self, *args, **kwargs): ...  # pragma: no cove
+    def __call__(self, *args, **kwargs): ...  # pragma: no cov
 
 
 ReturnTagFunc = Callable[P, TagFunc]
 DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
 
 
-def tag(name: str, alias: str | None = None) -> DecoratorTagFunc:
+def tag(
+    name: str, alias: str | None = None
+) -> DecoratorTagFunc:  # pragma: no cov
     """Tag decorator function that set function attributes, ``tag`` and ``name``
     for making registries variable.
 
@@ -364,7 +161,7 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    for module in load_config().engine.registry:
+    for module in config.regis_hook:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(f"{module}.{submodule}")
@@ -396,9 +193,14 @@ def make_registry(submodule: str) -> dict[str, Registry]:
 class BaseParam(BaseModel, ABC):
     """Base Parameter that use to make Params Model."""
 
-    desc: Optional[str] = None
-    required: bool = True
-    type: str
+    desc: Optional[str] = Field(
+        default=None, description="A description of parameter providing."
+    )
+    required: bool = Field(
+        default=True,
+        description="A require flag that force to pass this parameter value.",
+    )
+    type: str = Field(description="A type of parameter.")
 
     @abstractmethod
     def receive(self, value: Optional[Any] = None) -> Any:
@@ -406,19 +208,20 @@ class BaseParam(BaseModel, ABC):
             "Receive value and validate typing before return valid value."
         )
 
-    @field_serializer("type")
-    def __serializer_type(self, value: str) -> str:
-        """Serialize the value of the type field.
-
-        :rtype: str
-        """
-        return value
-
 
 class DefaultParam(BaseParam):
-    """Default Parameter that will check default if it required"""
+    """Default Parameter that will check default if it required. This model do
+    not implement the receive method.
+    """
 
-    default: Optional[str] = None
+    required: bool = Field(
+        default=False,
+        description="A require flag for the default-able parameter value.",
+    )
+    default: Optional[str] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
 
     @abstractmethod
     def receive(self, value: Optional[Any] = None) -> Any:
@@ -426,22 +229,12 @@ class DefaultParam(BaseParam):
             "Receive value and validate typing before return valid value."
         )
 
-    @model_validator(mode="after")
-    def __check_default(self) -> Self:
-        """Check default value should pass when it set required."""
-        if not self.required and self.default is None:
-            raise ParamValueException(
-                "Default should set when this parameter does not required."
-            )
-        return self
-
 
 class DatetimeParam(DefaultParam):
     """Datetime parameter."""
 
     type: Literal["datetime"] = "datetime"
-    required: bool = False
-    default: datetime = Field(default_factory=dt_now)
+    default: datetime = Field(default_factory=get_dt_now)
 
     def receive(self, value: str | datetime | date | None = None) -> datetime:
         """Receive value that match with datetime. If a input value pass with
@@ -463,7 +256,12 @@ class DatetimeParam(DefaultParam):
                 f"Value that want to convert to datetime does not support for "
                 f"type: {type(value)}"
             )
-        return datetime.fromisoformat(value)
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            raise ParamValueException(
+                f"Invalid isoformat string: {value!r}"
+            ) from None
 
 
 class StrParam(DefaultParam):
@@ -471,7 +269,7 @@ class StrParam(DefaultParam):
 
     type: Literal["str"] = "str"
 
-    def receive(self, value: Optional[str] = None) -> str | None:
+    def receive(self, value: str | None = None) -> str | None:
         """Receive value that match with str.
 
         :param value: A value that want to validate with string parameter type.
@@ -486,8 +284,12 @@ class IntParam(DefaultParam):
     """Integer parameter."""
 
     type: Literal["int"] = "int"
+    default: Optional[int] = Field(
+        default=None,
+        description="A default value if parameter does not pass.",
+    )
 
-    def receive(self, value: Optional[int] = None) -> int | None:
+    def receive(self, value: int | None = None) -> int | None:
         """Receive value that match with int.
 
         :param value: A value that want to validate with integer parameter type.
@@ -498,10 +300,9 @@ class IntParam(DefaultParam):
         if not isinstance(value, int):
             try:
                 return int(str(value))
-            except TypeError as err:
+            except ValueError as err:
                 raise ParamValueException(
-                    f"Value that want to convert to integer does not support "
-                    f"for type: {type(value)}"
+                    f"Value can not convert to int, {value}, with base 10"
                 ) from err
         return value
 
@@ -510,15 +311,19 @@ class ChoiceParam(BaseParam):
     """Choice parameter."""
 
     type: Literal["choice"] = "choice"
-    options: list[str]
+    options: list[str] = Field(description="A list of choice parameters.")
+
+    def receive(self, value: str | None = None) -> str:
+        """Receive value that match with options.
 
-    def receive(self, value: Optional[str] = None) -> str:
-        """Receive value that match with options."""
+        :param value: A value that want to select from the options field.
+        :rtype: str
+        """
         # NOTE:
         #   Return the first value in options if does not pass any input value
         if value is None:
             return self.options[0]
-        if any(value not in self.options):
+        if value not in self.options:
             raise ParamValueException(
                 f"{value!r} does not match any value in choice options."
             )
@@ -545,7 +350,7 @@ class Result:
 
     status: int = field(default=2)
     context: DictData = field(default_factory=dict)
-    start_at: datetime = field(default_factory=dt_now, compare=False)
+    start_at: datetime = field(default_factory=get_dt_now, compare=False)
     end_at: Optional[datetime] = field(default=None, compare=False)
 
     # NOTE: Ignore this field to compare another result model with __eq__.
@@ -577,15 +382,15 @@ class Result:
         :param running_id: A running ID that want to update on this model.
         :rtype: Self
         """
-        self._parent_run_id = running_id
+        self._parent_run_id: str = running_id
         return self
 
     @property
-    def parent_run_id(self):
+    def parent_run_id(self) -> str:
         return self._parent_run_id
 
     @property
-    def run_id(self):
+    def run_id(self) -> str:
         return self._run_id
 
     def catch(self, status: int, context: DictData) -> Self:
@@ -622,8 +427,8 @@ class Result:
         self.__dict__["context"]["jobs"].update(result.context)
 
         # NOTE: Update running ID from an incoming result.
-        self._parent_run_id = result.parent_run_id
-        self._run_id = result.run_id
+        self._parent_run_id: str = result.parent_run_id
+        self._run_id: str = result.run_id
         return self
 
 
@@ -684,7 +489,7 @@ def make_filter_registry() -> dict[str, FilterRegistry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    for module in load_config().engine.registry_filter:
+    for module in config.regis_filter:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -716,11 +521,11 @@ def get_args_const(
         raise UtilException(
             f"Post-filter: {expr} does not valid because it raise syntax error."
         ) from None
-    body: list[Expr] = mod.body
 
+    body: list[Expr] = mod.body
     if len(body) > 1:
         raise UtilException(
-            "Post-filter function should be only one calling per wf"
+            "Post-filter function should be only one calling per workflow."
         )
 
     caller: Union[Name, Call]
@@ -736,12 +541,15 @@ def get_args_const(
     keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
 
     if any(not isinstance(i, Constant) for i in args):
-        raise UtilException("Argument should be constant.")
+        raise UtilException(f"Argument of {expr} should be constant.")
+
+    if any(not isinstance(i, Constant) for i in keywords.values()):
+        raise UtilException(f"Keyword argument of {expr} should be constant.")
 
     return name.id, args, keywords
 
 
-@custom_filter("fmt")
+@custom_filter("fmt")  # pragma: no cov
 def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
     """Format datetime object to string with the format."""
     if isinstance(value, datetime):
@@ -752,16 +560,18 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 
 
 def map_post_filter(
-    value: Any,
+    value: T,
     post_filter: list[str],
     filters: dict[str, FilterRegistry],
-) -> Any:
+) -> T:
     """Mapping post-filter to value with sequence list of filter function name
     that will get from the filter registry.
 
     :param value: A string value that want to mapped with filter function.
     :param post_filter: A list of post-filter function name.
     :param filters: A filter registry.
+
+    :rtype: T
     """
     for _filter in post_filter:
         func_name, _args, _kwargs = get_args_const(_filter)
@@ -784,6 +594,8 @@ def map_post_filter(
                     value: Any = func(value)
             else:
                 value: Any = f_func(value, *args, **kwargs)
+        except UtilException:
+            raise
         except Exception as err:
             logger.warning(str(err))
             raise UtilException(
@@ -796,8 +608,8 @@ def map_post_filter(
 def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
     """Check value should not pass template with not_in value prefix.
 
-    :param value:
-    :param not_in:
+    :param value: A value that want to find parameter template prefix.
+    :param not_in: The not in string that use in the `.startswith` function.
     :rtype: bool
     """
     if isinstance(value, dict):
@@ -815,7 +627,7 @@ def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
 def has_template(value: Any) -> bool:
     """Check value include templating string.
 
-    :param value:
+    :param value: A value that want to find parameter template.
     :rtype: bool
     """
     if isinstance(value, dict):
@@ -971,6 +783,7 @@ def batch(iterable: Iterator[Any], n: int) -> Iterator[Any]:
     """
     if n < 1:
         raise ValueError("n must be at least one")
+
     it: Iterator[Any] = iter(iterable)
     while True:
         chunk_it = islice(it, n)