ddeutil-workflow 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.1"
+__version__: str = "0.0.3"

ddeutil/workflow/__types.py

@@ -9,3 +9,4 @@ from typing import Any
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
+DictStr = dict[str, str]

ddeutil/workflow/conn.py

@@ -10,7 +10,7 @@ from collections.abc import Iterator
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, TypeVar
 
-from ddeutil.model.conn import Conn as ConnModel
+from ddeutil.io.models.conn import Conn as ConnModel
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator
 from pydantic.types import SecretStr
@@ -43,27 +43,21 @@ class BaseConn(BaseModel):
     ]
 
     @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
-        """Construct Connection with Loader object with specific config name.
+    def from_dict(cls, values: DictData) -> Self:
+        """Construct Connection Model from dict data. This construct is
+        different with ``.model_validate()`` because it will prepare the values
+        before using it if the data dose not have 'url'.
 
-        :param name:
-        :param externals:
+        :param values: A dict data that use to construct this model.
         """
-        loader: Loader = Loader(name, externals=externals)
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        # NOTE: filter out the fields of this model.
         filter_data: DictData = {
-            k: loader.data.pop(k)
-            for k in loader.data.copy()
+            k: values.pop(k)
+            for k in values.copy()
             if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
         }
-        if "url" in loader.data:
-            url: ConnModel = ConnModel.from_url(loader.data.pop("url"))
+        if "url" in values:
+            url: ConnModel = ConnModel.from_url(values.pop("url"))
             return cls(
                 dialect=url.dialect,
                 host=url.host,
@@ -73,27 +67,38 @@ class BaseConn(BaseModel):
                 # NOTE:
                 #   I will replace None endpoint with memory value for SQLite
                 #   connection string.
-                endpoint=cls.__prepare_slash_from_url(url.endpoint or "memory"),
+                endpoint=(url.endpoint or "memory"),
                 # NOTE: This order will show that externals this the top level.
-                extras=(url.options | filter_data | externals),
+                extras=(url.options | filter_data),
             )
         return cls.model_validate(
             obj={
-                "extras": (
-                    loader.data.pop("extras", {}) | filter_data | externals
-                ),
-                **loader.data,
+                "extras": (values.pop("extras", {}) | filter_data),
+                **values,
             }
         )
 
     @classmethod
-    def __prepare_slash_from_url(cls, value: str) -> str:
-        if value.startswith("/"):
-            return value[1:]
-        return value
+    def from_loader(cls, name: str, externals: DictData) -> Self:
+        """Construct Connection with Loader object with specific config name.
+
+        :param name: A config name.
+        :param externals: A external data that want to adding to extras.
+        """
+        loader: Loader = Loader(name, externals=externals)
+        # NOTE: Validate the config type match with current connection model
+        if loader.type != cls:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        return cls.from_dict(
+            {
+                "extras": (loader.data.pop("extras", {}) | externals),
+                **loader.data,
+            }
+        )
 
     @field_validator("endpoint")
     def __prepare_slash(cls, value: str) -> str:
+        """Prepare slash character that map double form URL model loading."""
         if value.startswith("//"):
             return value[1:]
         return value
@@ -146,7 +151,7 @@ class SFTP(Conn):
     dialect: Literal["sftp"] = "sftp"
 
     def __client(self):
-        from .vendors.sftp_wrapped import WrapSFTP
+        from .vendors.sftp import WrapSFTP
 
         return WrapSFTP(
             host=self.host,

ddeutil/workflow/exceptions.py

@@ -8,75 +8,5 @@ Define Errors Object for Node package
 """
 from __future__ import annotations
 
-from typing import Union
-
-
-class BaseError(Exception):
-    """Base Error Object that use for catch any errors statement of
-    all step in this src
-    """
-
-
-class WorkflowBaseError(BaseError):
-    """Core Base Error object"""
-
-
-class ConfigNotFound(WorkflowBaseError):
-    """Error raise for a method not found the config file or data."""
-
-
-class ConfigArgumentError(WorkflowBaseError):
-    """Error raise for a wrong configuration argument."""
-
-    def __init__(self, argument: Union[str, tuple], message: str):
-        """Main Initialization that merge the argument and message input values
-        with specific content message together like
-
-            `__class__` with `argument`, `message`
-
-        :param argument: Union[str, tuple]
-        :param message: str
-        """
-        if isinstance(argument, tuple):
-            _last_arg: str = str(argument[-1])
-            _argument: str = (
-                (
-                    ", ".join([f"{_!r}" for _ in argument[:-1]])
-                    + f", and {_last_arg!r}"
-                )
-                if len(argument) > 1
-                else f"{_last_arg!r}"
-            )
-        else:
-            _argument: str = f"{argument!r}"
-        _message: str = f"with {_argument}, {message}"
-        super().__init__(_message)
-
-
-class ConnArgumentError(ConfigArgumentError):
-    """Error raise for wrong connection argument when loading or parsing"""
-
-
-class DsArgumentError(ConfigArgumentError):
-    """Error raise for wrong catalog argument when loading or parsing"""
-
-
-class NodeArgumentError(ConfigArgumentError):
-    """Error raise for wrong node argument when loading or parsing"""
-
-
-class ScdlArgumentError(ConfigArgumentError):
-    """Error raise for wrong schedule argument when loading or parsing"""
-
-
-class PipeArgumentError(ConfigArgumentError):
-    """Error raise for wrong pipeline argument when loading or parsing"""
-
-
-class PyException(Exception): ...
-
-
-class ShellException(Exception): ...
-
 
 class TaskException(Exception): ...

ddeutil/workflow/loader.py

@@ -5,187 +5,47 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import copy
-import logging
-import urllib.parse
 from functools import cached_property
-from typing import Any, Callable, TypeVar
+from typing import Any, ClassVar, TypeVar
 
 from ddeutil.core import (
-    clear_cache,
     getdot,
     hasdot,
     import_string,
-    setdot,
 )
 from ddeutil.io import (
-    ConfigNotFound,
-    Params,
+    PathData,
     PathSearch,
-    Register,
     YamlEnvFl,
-    map_func,
 )
-from ddeutil.io.__conf import UPDATE_KEY, VERSION_KEY
-from fmtutil import Datetime
-from pydantic import BaseModel
-from typing_extensions import Self
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
 
 from .__regex import RegexConf
-from .__types import DictData, TupleStr
-from .exceptions import ConfigArgumentError
+from .__types import DictData
 
+T = TypeVar("T")
+BaseModelType = type[BaseModel]
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 
 
-class YamlEnvQuote(YamlEnvFl):
+class Engine(BaseModel):
+    """Engine Model"""
 
-    @staticmethod
-    def prepare(x: str) -> str:
-        return urllib.parse.quote_plus(str(x))
+    paths: PathData = Field(default_factory=PathData)
+    registry: list[str] = Field(default_factory=lambda: ["ddeutil.workflow"])
 
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        return values
 
-class BaseLoad:
-    """Base configuration data loading object for load config data from
-    `cls.load_stage` stage. The base loading object contain necessary
-    properties and method for type object.
 
-    :param data: dict : A configuration data content with fix keys, `name`,
-        `fullname`, and `data`.
-    :param params: Optional[dict] : A parameters mapping for some
-        subclass of loading use.
-    """
-
-    # NOTE: Set loading config for inherit
-    load_prefixes: TupleStr = ("conn",)
-    load_datetime_name: str = "audit_date"
-    load_datetime_fmt: str = "%Y-%m-%d %H:%M:%S"
-
-    # NOTE: Set preparing config for inherit
-    data_excluded: TupleStr = (UPDATE_KEY, VERSION_KEY)
-    option_key: TupleStr = ("parameters",)
-    datetime_key: TupleStr = ("endpoint",)
-
-    @classmethod
-    def from_register(
-        cls,
-        name: str,
-        params: Params,
-        externals: DictData | None = None,
-    ) -> Self:
-        """Loading config data from register object.
-
-        :param name: A name of config data catalog that can register.
-        :type name: str
-        :param params: A params object.
-        :type params: Params
-        :param externals: A external parameters
-        :type externals: DictData | None(=None)
-        """
-        try:
-            rs: Register = Register(
-                name=name,
-                stage=params.stage_final,
-                params=params,
-                loader=YamlEnvQuote,
-            )
-        except ConfigNotFound:
-            rs: Register = Register(
-                name=name,
-                params=params,
-                loader=YamlEnvQuote,
-            ).deploy(stop=params.stage_final)
-        return cls(
-            name=rs.name,
-            data=rs.data().copy(),
-            params=params,
-            externals=externals,
-        )
-
-    def __init__(
-        self,
-        name: str,
-        data: DictData,
-        params: Params,
-        externals: DictData | None = None,
-    ) -> None:
-        """Main initialize base config object which get a name of configuration
-        and load data by the register object.
-        """
-        self.name: str = name
-        self.__data: DictData = data
-        self.params: Params = params
-        self.externals: DictData = externals or {}
-
-        # NOTE: Validate step of base loading object.
-        if not any(
-            self.name.startswith(prefix) for prefix in self.load_prefixes
-        ):
-            raise ConfigArgumentError(
-                "prefix",
-                (
-                    f"{self.name!r} does not starts with the "
-                    f"{self.__class__.__name__} prefixes: "
-                    f"{self.load_prefixes!r}."
-                ),
-            )
-
-    @property
-    def updt(self):
-        return self.data.get(UPDATE_KEY)
-
-    @cached_property
-    def _map_data(self) -> DictData:
-        """Return configuration data without key in the excluded key set."""
-        data: DictData = self.__data.copy()
-        rs: DictData = {k: data[k] for k in data if k not in self.data_excluded}
-
-        # Mapping datetime format to string value.
-        for _ in self.datetime_key:
-            if hasdot(_, rs):
-                # Fill format datetime object to any type value.
-                rs: DictData = setdot(
-                    _,
-                    rs,
-                    map_func(
-                        getdot(_, rs),
-                        Datetime.parse(
-                            value=self.externals[self.load_datetime_name],
-                            fmt=self.load_datetime_fmt,
-                        ).format,
-                    ),
-                )
-        return rs
+class Params(BaseModel):
+    """Params Model"""
 
-    @property
-    def data(self) -> DictData:
-        """Return deep copy of the input data.
-
-        :rtype: DictData
-        """
-        return copy.deepcopy(self._map_data)
-
-    @clear_cache(attrs=("type", "_map_data"))
-    def refresh(self) -> Self:
-        """Refresh configuration data. This process will use `deploy` method
-        of the register object.
-
-        :rtype: Self
-        """
-        return self.from_register(
-            name=self.name,
-            params=self.params,
-            externals=self.externals,
-        )
-
-    @cached_property
-    def type(self) -> Any:
-        """Return object type which implement in `config_object` key."""
-        if not (_typ := self.data.get("type")):
-            raise ValueError(
-                f"the 'type' value: {_typ} does not exists in config data."
-            )
-        return import_string(f"ddeutil.pipe.{_typ}")
+    engine: Engine = Field(default_factory=Engine)
 
 
 class SimLoad:
@@ -195,13 +55,11 @@ class SimLoad:
     :param params: A Params model object.
     :param externals: An external parameters
 
-    Note:
+    Noted:
         The config data should have ``type`` key for engine can know what is
     config should to do next.
     """
 
-    import_prefix: str = "ddeutil.workflow"
-
     def __init__(
         self,
         name: str,
@@ -215,7 +73,7 @@ class SimLoad:
             ):
                 self.data = data
         if not self.data:
-            raise ConfigNotFound(f"Config {name!r} does not found on conf path")
+            raise ValueError(f"Config {name!r} does not found on conf path")
         self.__conf_params: Params = params
         self.externals: DictData = externals
 
@@ -224,7 +82,7 @@ class SimLoad:
         return self.__conf_params
 
     @cached_property
-    def type(self) -> AnyModel:
+    def type(self) -> BaseModelType:
         """Return object type which implement in `config_object` key."""
         if not (_typ := self.data.get("type")):
             raise ValueError(
@@ -234,33 +92,25 @@ class SimLoad:
             # NOTE: Auto adding module prefix if it does not set
             return import_string(f"ddeutil.workflow.{_typ}")
         except ModuleNotFoundError:
+            for registry in self.conf_params.engine.registry:
+                try:
+                    return import_string(f"{registry}.{_typ}")
+                except ModuleNotFoundError:
+                    continue
             return import_string(f"{_typ}")
 
-    def params(self) -> dict[str, Callable[[Any], Any]]:
-        """Return a mapping of key from params and imported value on params."""
-        if not (p := self.data.get("params", {})):
-            return p
+    def load(self) -> AnyModel:
+        return self.type.model_validate(self.data)
 
-        try:
-            return {i: import_string(f"{self.import_prefix}.{p[i]}") for i in p}
-        except ModuleNotFoundError as err:
-            logging.error(err)
-            raise err
 
-    def validate_params(self, param: dict[str, Any]) -> dict[str, Any]:
-        """Return parameter that want to catch before workflow running."""
-        try:
-            return {i: caller(param[i]) for i, caller in self.params().items()}
-        except KeyError as err:
-            logging.error(f"Parameter: {err} does not exists from passing")
-            raise err
-        except ValueError as err:
-            logging.error("Value that passing to params does not valid")
-            raise err
+class Loader(SimLoad):
+    """Main Loader Object that get the config `yaml` file from current path.
 
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param externals: An external parameters
+    """
 
-class Loader(SimLoad):
-    """Main Loader Object."""
+    conf_name: ClassVar[str] = "workflows-conf"
 
     def __init__(
         self,
@@ -278,23 +128,37 @@ class Loader(SimLoad):
 
     @classmethod
     def config(cls, path: str | None = None) -> Params:
+        """Load Config data from ``workflows-conf.yaml`` file."""
         return Params.model_validate(
-            YamlEnvFl(path or "./workflows-conf.yaml").read()
+            YamlEnvFl(path or f"./{cls.conf_name}.yaml").read()
         )
 
 
-def map_caller(value: str, params: dict[str, Any]) -> Any:
-    """Map caller value that found from ``RE_CALLER`` regex.
+def map_params(value: Any, params: dict[str, Any]) -> Any:
+    """Map caller value that found from ``RE_CALLER`` regular expression.
+
+    :param value: A value that want to mapped with an params
+    :param params: A parameter value that getting with matched regular
+        expression.
 
-    :returns: Any value that getter of caller receive from the params.
+    :rtype: Any
+    :returns: An any getter value from the params input.
     """
+    if isinstance(value, dict):
+        return {k: map_params(value[k], params) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([map_params(i, params) for i in value])
+    elif not isinstance(value, str):
+        return value
+
     if not (found := RegexConf.RE_CALLER.search(value)):
         return value
+
     # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
-    caller = found.group("caller")
+    caller: str = found.group("caller")
     if not hasdot(caller, params):
         raise ValueError(f"params does not set caller: {caller!r}")
-    getter = getdot(caller, params)
+    getter: Any = getdot(caller, params)
 
     # NOTE: check type of vars
     if isinstance(getter, (str, int)):