ddeutil-workflow 0.0.2__tar.gz → 0.0.3__tar.gz

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.2
+Version: 0.0.3
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -83,7 +83,13 @@ Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
 The main feature of this project is the `Pipeline` object that can call any
-registried function.
+registries function. The pipeline can handle everything that you want to do, it
+will passing parameters and catching the output for re-use it to next step.
+
+> [!IMPORTANT]
+> In the future of this project, I will drop the connection and dataset to
+> dynamic registries instead of main features because it have a lot of maintain
+> vendor codes and deps. (I do not have time to handle this features)
 
 ### Connection
 
@@ -104,7 +110,9 @@ assert conn.ping()
 
 ### Dataset
 
-The dataset is define any objects on the connection.
+The dataset is define any objects on the connection. This feature was implemented
+on `/vendors` because it has a lot of tools that can interact with any data systems
+in the data tool stacks.
 
 ```yaml
 ds_postgres_customer_tbl:
@@ -116,7 +124,7 @@ ds_postgres_customer_tbl:
 ```
 
 ```python
-from ddeutil.workflow.dataset import PostgresTbl
+from ddeutil.workflow.vendors.pg import PostgresTbl
 
 dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
 assert dataset.exists()
@@ -126,14 +134,14 @@ assert dataset.exists()
 
 ```yaml
 schd_for_node:
-  type: schedule.Scdl
+  type: schedule.Schedule
   cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.schedule import Scdl
+from ddeutil.workflow.schedule import Schedule
 
-scdl = Scdl.from_loader(name='schd_for_node', externals={})
+scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
 
 cron_iterate = scdl.generate('2022-01-01 00:00:00')
@@ -234,14 +242,10 @@ pipe_el_pg_to_lake:
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date:
-      type: datetime
-    sp_name:
-      type: str
-    source_name:
-      type: str
-    target_name:
-      type: str
+    run_date: datetime
+    sp_name: str
+    source_name: str
+    target_name: str
   jobs:
     transform:
       stages:

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/README.md

@@ -48,7 +48,13 @@ Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
 The main feature of this project is the `Pipeline` object that can call any
-registried function.
+registries function. The pipeline can handle everything that you want to do, it
+will passing parameters and catching the output for re-use it to next step.
+
+> [!IMPORTANT]
+> In the future of this project, I will drop the connection and dataset to
+> dynamic registries instead of main features because it have a lot of maintain
+> vendor codes and deps. (I do not have time to handle this features)
 
 ### Connection
 
@@ -69,7 +75,9 @@ assert conn.ping()
 
 ### Dataset
 
-The dataset is define any objects on the connection.
+The dataset is define any objects on the connection. This feature was implemented
+on `/vendors` because it has a lot of tools that can interact with any data systems
+in the data tool stacks.
 
 ```yaml
 ds_postgres_customer_tbl:
@@ -81,7 +89,7 @@ ds_postgres_customer_tbl:
 ```
 
 ```python
-from ddeutil.workflow.dataset import PostgresTbl
+from ddeutil.workflow.vendors.pg import PostgresTbl
 
 dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
 assert dataset.exists()
@@ -91,14 +99,14 @@ assert dataset.exists()
 
 ```yaml
 schd_for_node:
-  type: schedule.Scdl
+  type: schedule.Schedule
   cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.schedule import Scdl
+from ddeutil.workflow.schedule import Schedule
 
-scdl = Scdl.from_loader(name='schd_for_node', externals={})
+scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
 
 cron_iterate = scdl.generate('2022-01-01 00:00:00')
@@ -199,14 +207,10 @@ pipe_el_pg_to_lake:
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date:
-      type: datetime
-    sp_name:
-      type: str
-    source_name:
-      type: str
-    target_name:
-      type: str
+    run_date: datetime
+    sp_name: str
+    source_name: str
+    target_name: str
   jobs:
     transform:
       stages:

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/pyproject.toml

@@ -79,6 +79,10 @@ addopts = [
     "--strict-markers",
 ]
 filterwarnings = ["error"]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 
 [tool.black]
 line-length = 80
@@ -97,7 +101,6 @@ exclude = """
         | build
         | dist
         | venv
-        | __legacy
     )/
 )
 """
@@ -113,7 +116,6 @@ exclude = [
     "build",
     "dist",
     "venv",
-    "__legacy",
 ]
 
 [tool.ruff.lint]

ddeutil_workflow-0.0.3/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.3"

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/src/ddeutil/workflow/__types.py

@@ -9,3 +9,4 @@ from typing import Any
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
+DictStr = dict[str, str]

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/src/ddeutil/workflow/conn.py

@@ -43,8 +43,14 @@ class BaseConn(BaseModel):
     ]
 
     @classmethod
-    def from_dict(cls, values: DictData):
-        """Construct Connection with dict of data"""
+    def from_dict(cls, values: DictData) -> Self:
+        """Construct Connection Model from dict data. This construct is
+        different with ``.model_validate()`` because it will prepare the values
+        before using it if the data dose not have 'url'.
+
+        :param values: A dict data that use to construct this model.
+        """
+        # NOTE: filter out the fields of this model.
         filter_data: DictData = {
             k: values.pop(k)
             for k in values.copy()
@@ -73,15 +79,11 @@ class BaseConn(BaseModel):
         )
 
     @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
+    def from_loader(cls, name: str, externals: DictData) -> Self:
         """Construct Connection with Loader object with specific config name.
 
-        :param name:
-        :param externals:
+        :param name: A config name.
+        :param externals: A external data that want to adding to extras.
         """
         loader: Loader = Loader(name, externals=externals)
         # NOTE: Validate the config type match with current connection model
@@ -96,6 +98,7 @@ class BaseConn(BaseModel):
 
     @field_validator("endpoint")
     def __prepare_slash(cls, value: str) -> str:
+        """Prepare slash character that map double form URL model loading."""
         if value.startswith("//"):
             return value[1:]
         return value
@@ -148,7 +151,7 @@ class SFTP(Conn):
     dialect: Literal["sftp"] = "sftp"
 
     def __client(self):
-        from .vendors.sftp_wrapped import WrapSFTP
+        from .vendors.sftp import WrapSFTP
 
         return WrapSFTP(
             host=self.host,

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/src/ddeutil/workflow/exceptions.py

@@ -9,24 +9,4 @@ Define Errors Object for Node package
 from __future__ import annotations
 
 
-class BaseError(Exception):
-    """Base Error Object that use for catch any errors statement of
-    all step in this src
-    """
-
-
-class WorkflowBaseError(BaseError):
-    """Core Base Error object"""
-
-
-class ConfigNotFound(WorkflowBaseError):
-    """Error raise for a method not found the config file or data."""
-
-
-class PyException(Exception): ...
-
-
-class ShellException(Exception): ...
-
-
 class TaskException(Exception): ...

{ddeutil_workflow-0.0.2 → ddeutil_workflow-0.0.3}/src/ddeutil/workflow/loader.py

@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import Any, TypeVar
+from typing import Any, ClassVar, TypeVar
 
 from ddeutil.core import (
     getdot,
@@ -14,12 +14,12 @@ from ddeutil.core import (
     import_string,
 )
 from ddeutil.io import (
-    ConfigNotFound,
-    Params,
+    PathData,
     PathSearch,
     YamlEnvFl,
 )
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
 
 from .__regex import RegexConf
 from .__types import DictData
@@ -29,6 +29,25 @@ BaseModelType = type[BaseModel]
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 
 
+class Engine(BaseModel):
+    """Engine Model"""
+
+    paths: PathData = Field(default_factory=PathData)
+    registry: list[str] = Field(default_factory=lambda: ["ddeutil.workflow"])
+
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        return values
+
+
+class Params(BaseModel):
+    """Params Model"""
+
+    engine: Engine = Field(default_factory=Engine)
+
+
 class SimLoad:
     """Simple Load Object that will search config data by name.
 
@@ -36,13 +55,11 @@ class SimLoad:
     :param params: A Params model object.
     :param externals: An external parameters
 
-    Note:
+    Noted:
         The config data should have ``type`` key for engine can know what is
     config should to do next.
     """
 
-    import_prefix: str = "ddeutil.workflow"
-
     def __init__(
         self,
         name: str,
@@ -56,7 +73,7 @@ class SimLoad:
             ):
                 self.data = data
         if not self.data:
-            raise ConfigNotFound(f"Config {name!r} does not found on conf path")
+            raise ValueError(f"Config {name!r} does not found on conf path")
         self.__conf_params: Params = params
         self.externals: DictData = externals
 
@@ -75,6 +92,11 @@ class SimLoad:
             # NOTE: Auto adding module prefix if it does not set
             return import_string(f"ddeutil.workflow.{_typ}")
         except ModuleNotFoundError:
+            for registry in self.conf_params.engine.registry:
+                try:
+                    return import_string(f"{registry}.{_typ}")
+                except ModuleNotFoundError:
+                    continue
             return import_string(f"{_typ}")
 
     def load(self) -> AnyModel:
@@ -82,12 +104,14 @@ class SimLoad:
 
 
 class Loader(SimLoad):
-    """Main Loader Object.
+    """Main Loader Object that get the config `yaml` file from current path.
 
     :param name: A name of config data that will read by Yaml Loader object.
     :param externals: An external parameters
     """
 
+    conf_name: ClassVar[str] = "workflows-conf"
+
     def __init__(
         self,
         name: str,
@@ -106,12 +130,16 @@ class Loader(SimLoad):
     def config(cls, path: str | None = None) -> Params:
         """Load Config data from ``workflows-conf.yaml`` file."""
         return Params.model_validate(
-            YamlEnvFl(path or "./workflows-conf.yaml").read()
+            YamlEnvFl(path or f"./{cls.conf_name}.yaml").read()
         )
 
 
 def map_params(value: Any, params: dict[str, Any]) -> Any:
-    """Map caller value that found from ``RE_CALLER`` regex.
+    """Map caller value that found from ``RE_CALLER`` regular expression.
+
+    :param value: A value that want to mapped with an params
+    :param params: A parameter value that getting with matched regular
+        expression.
 
     :rtype: Any
     :returns: An any getter value from the params input.