ddeutil-workflow 0.0.1__tar.gz → 0.0.2__tar.gz

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.1
+Version: 0.0.2
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -23,15 +23,14 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
-Requires-Dist: ddeutil-model
 Requires-Dist: python-dotenv
 Provides-Extra: test
 Requires-Dist: sqlalchemy==2.0.30; extra == "test"
 Requires-Dist: paramiko==3.4.0; extra == "test"
 Requires-Dist: sshtunnel==0.4.0; extra == "test"
-Requires-Dist: boto3==1.34.109; extra == "test"
+Requires-Dist: boto3==1.34.117; extra == "test"
 Requires-Dist: fsspec==2024.5.0; extra == "test"
-Requires-Dist: polars==0.20.26; extra == "test"
+Requires-Dist: polars==0.20.31; extra == "test"
 Requires-Dist: pyarrow==16.1.0; extra == "test"
 
 # Data Utility: _Workflow_
@@ -83,6 +82,9 @@ The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
+The main feature of this project is the `Pipeline` object that can call any
+registried function.
+
 ### Connection
 
 The connection for worker able to do any thing.
@@ -155,8 +157,10 @@ The state of doing lists that worker should to do. It be collection of the stage
 run_py_local:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    author-run: utils.receive.string
-    run-date: utils.receive.datetime
+    author-run:
+      type: str
+    run-date:
+      type: datetime
   jobs:
     first-job:
       stages:
@@ -203,13 +207,15 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 pipe_el_pg_to_lake:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run-date: utils.receive.datetime
-    author-email: utils.receive.string
+    run-date:
+      type: datetime
+    author-email:
+      type: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
-          id: extract
+          id: extract-load
           task: tasks/postgres-to-delta@polars
           with:
             source:
@@ -222,21 +228,26 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Hooks (Transform)
+### Tasks (Transform)
 
 ```yaml
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date: utils.receive.datetime
-    sp_name: utils.receive.string
-    source_name: utils.receive.string
-    target_name: utils.receive.string
+    run_date:
+      type: datetime
+    sp_name:
+      type: str
+    source_name:
+      type: str
+    target_name:
+      type: str
   jobs:
     transform:
       stages:
         - name: "Transform Data in MS SQL Server"
-          hook: hooks/mssql-proc@odbc
+          id: transform
+          task: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.2}/README.md

@@ -47,6 +47,9 @@ The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
+The main feature of this project is the `Pipeline` object that can call any
+registried function.
+
 ### Connection
 
 The connection for worker able to do any thing.
@@ -119,8 +122,10 @@ The state of doing lists that worker should to do. It be collection of the stage
 run_py_local:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    author-run: utils.receive.string
-    run-date: utils.receive.datetime
+    author-run:
+      type: str
+    run-date:
+      type: datetime
   jobs:
     first-job:
       stages:
@@ -167,13 +172,15 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 pipe_el_pg_to_lake:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run-date: utils.receive.datetime
-    author-email: utils.receive.string
+    run-date:
+      type: datetime
+    author-email:
+      type: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
-          id: extract
+          id: extract-load
           task: tasks/postgres-to-delta@polars
           with:
             source:
@@ -186,21 +193,26 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Hooks (Transform)
+### Tasks (Transform)
 
 ```yaml
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date: utils.receive.datetime
-    sp_name: utils.receive.string
-    source_name: utils.receive.string
-    target_name: utils.receive.string
+    run_date:
+      type: datetime
+    sp_name:
+      type: str
+    source_name:
+      type: str
+    target_name:
+      type: str
   jobs:
     transform:
       stages:
         - name: "Transform Data in MS SQL Server"
-          hook: hooks/mssql-proc@odbc
+          id: transform
+          task: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.2}/pyproject.toml

@@ -28,7 +28,6 @@ requires-python = ">=3.9.13"
 dependencies = [
     "fmtutil",
     "ddeutil-io",
-    "ddeutil-model",
     "python-dotenv",
 ]
 dynamic = ["version"]
@@ -40,15 +39,15 @@ Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
 [project.optional-dependencies]
 test = [
     "sqlalchemy==2.0.30",
-    # SFTP warpper
+    # SFTP
     "paramiko==3.4.0",
     "sshtunnel==0.4.0",
-    # AWS
-    "boto3==1.34.109",
+    # AWS Client
+    "boto3==1.34.117",
     # Open files
     "fsspec==2024.5.0",
     # Polars
-    "polars==0.20.26",
+    "polars==0.20.31",
     "pyarrow==16.1.0",
 ]
 
@@ -133,4 +132,7 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"__init__.py" = ["F401"]
+"__init__.py" = [
+    "F401",
+    "F403",
+]

ddeutil_workflow-0.0.2/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.2"

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.2}/src/ddeutil/workflow/conn.py

@@ -10,7 +10,7 @@ from collections.abc import Iterator
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, TypeVar
 
-from ddeutil.model.conn import Conn as ConnModel
+from ddeutil.io.models.conn import Conn as ConnModel
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator
 from pydantic.types import SecretStr
@@ -43,27 +43,15 @@ class BaseConn(BaseModel):
     ]
 
     @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
-        """Construct Connection with Loader object with specific config name.
-
-        :param name:
-        :param externals:
-        """
-        loader: Loader = Loader(name, externals=externals)
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
+    def from_dict(cls, values: DictData):
+        """Construct Connection with dict of data"""
         filter_data: DictData = {
-            k: loader.data.pop(k)
-            for k in loader.data.copy()
+            k: values.pop(k)
+            for k in values.copy()
             if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
         }
-        if "url" in loader.data:
-            url: ConnModel = ConnModel.from_url(loader.data.pop("url"))
+        if "url" in values:
+            url: ConnModel = ConnModel.from_url(values.pop("url"))
             return cls(
                 dialect=url.dialect,
                 host=url.host,
@@ -73,24 +61,38 @@ class BaseConn(BaseModel):
                 # NOTE:
                 #   I will replace None endpoint with memory value for SQLite
                 #   connection string.
-                endpoint=cls.__prepare_slash_from_url(url.endpoint or "memory"),
+                endpoint=(url.endpoint or "memory"),
                 # NOTE: This order will show that externals this the top level.
-                extras=(url.options | filter_data | externals),
+                extras=(url.options | filter_data),
             )
         return cls.model_validate(
             obj={
-                "extras": (
-                    loader.data.pop("extras", {}) | filter_data | externals
-                ),
-                **loader.data,
+                "extras": (values.pop("extras", {}) | filter_data),
+                **values,
             }
         )
 
     @classmethod
-    def __prepare_slash_from_url(cls, value: str) -> str:
-        if value.startswith("/"):
-            return value[1:]
-        return value
+    def from_loader(
+        cls,
+        name: str,
+        externals: DictData,
+    ) -> Self:
+        """Construct Connection with Loader object with specific config name.
+
+        :param name:
+        :param externals:
+        """
+        loader: Loader = Loader(name, externals=externals)
+        # NOTE: Validate the config type match with current connection model
+        if loader.type != cls:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        return cls.from_dict(
+            {
+                "extras": (loader.data.pop("extras", {}) | externals),
+                **loader.data,
+            }
+        )
 
     @field_validator("endpoint")
     def __prepare_slash(cls, value: str) -> str:

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.2}/src/ddeutil/workflow/dataset.py

@@ -25,11 +25,7 @@ from .conn import SubclassConn
 from .loader import Loader
 
 EXCLUDED_EXTRAS: TupleStr = ("type",)
-OBJ_FMTS: FormatterGroupType = make_group(
-    {
-        "datetime": Datetime,
-    }
-)
+OBJ_FMTS: FormatterGroupType = make_group({"datetime": Datetime})
 
 
 class BaseDataset(BaseModel):

ddeutil_workflow-0.0.2/src/ddeutil/workflow/exceptions.py

@@ -0,0 +1,32 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""
+Define Errors Object for Node package
+"""
+from __future__ import annotations
+
+
+class BaseError(Exception):
+    """Base Error Object that use for catch any errors statement of
+    all step in this src
+    """
+
+
+class WorkflowBaseError(BaseError):
+    """Core Base Error object"""
+
+
+class ConfigNotFound(WorkflowBaseError):
+    """Error raise for a method not found the config file or data."""
+
+
+class PyException(Exception): ...
+
+
+class ShellException(Exception): ...
+
+
+class TaskException(Exception): ...

ddeutil_workflow-0.0.2/src/ddeutil/workflow/loader.py

@@ -0,0 +1,146 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from functools import cached_property
+from typing import Any, TypeVar
+
+from ddeutil.core import (
+    getdot,
+    hasdot,
+    import_string,
+)
+from ddeutil.io import (
+    ConfigNotFound,
+    Params,
+    PathSearch,
+    YamlEnvFl,
+)
+from pydantic import BaseModel
+
+from .__regex import RegexConf
+from .__types import DictData
+
+T = TypeVar("T")
+BaseModelType = type[BaseModel]
+AnyModel = TypeVar("AnyModel", bound=BaseModel)
+
+
+class SimLoad:
+    """Simple Load Object that will search config data by name.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param params: A Params model object.
+    :param externals: An external parameters
+
+    Note:
+        The config data should have ``type`` key for engine can know what is
+    config should to do next.
+    """
+
+    import_prefix: str = "ddeutil.workflow"
+
+    def __init__(
+        self,
+        name: str,
+        params: Params,
+        externals: DictData,
+    ) -> None:
+        self.data: DictData = {}
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in ("yml", "yaml")) and (
+                data := YamlEnvFl(file).read().get(name, {})
+            ):
+                self.data = data
+        if not self.data:
+            raise ConfigNotFound(f"Config {name!r} does not found on conf path")
+        self.__conf_params: Params = params
+        self.externals: DictData = externals
+
+    @property
+    def conf_params(self) -> Params:
+        return self.__conf_params
+
+    @cached_property
+    def type(self) -> BaseModelType:
+        """Return object type which implement in `config_object` key."""
+        if not (_typ := self.data.get("type")):
+            raise ValueError(
+                f"the 'type' value: {_typ} does not exists in config data."
+            )
+        try:
+            # NOTE: Auto adding module prefix if it does not set
+            return import_string(f"ddeutil.workflow.{_typ}")
+        except ModuleNotFoundError:
+            return import_string(f"{_typ}")
+
+    def load(self) -> AnyModel:
+        return self.type.model_validate(self.data)
+
+
+class Loader(SimLoad):
+    """Main Loader Object.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param externals: An external parameters
+    """
+
+    def __init__(
+        self,
+        name: str,
+        externals: DictData,
+        *,
+        path: str | None = None,
+    ) -> None:
+        self.data: DictData = {}
+
+        # NOTE: import params object from specific config file
+        params: Params = self.config(path)
+
+        super().__init__(name, params, externals)
+
+    @classmethod
+    def config(cls, path: str | None = None) -> Params:
+        """Load Config data from ``workflows-conf.yaml`` file."""
+        return Params.model_validate(
+            YamlEnvFl(path or "./workflows-conf.yaml").read()
+        )
+
+
+def map_params(value: Any, params: dict[str, Any]) -> Any:
+    """Map caller value that found from ``RE_CALLER`` regex.
+
+    :rtype: Any
+    :returns: An any getter value from the params input.
+    """
+    if isinstance(value, dict):
+        return {k: map_params(value[k], params) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([map_params(i, params) for i in value])
+    elif not isinstance(value, str):
+        return value
+
+    if not (found := RegexConf.RE_CALLER.search(value)):
+        return value
+
+    # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
+    caller: str = found.group("caller")
+    if not hasdot(caller, params):
+        raise ValueError(f"params does not set caller: {caller!r}")
+    getter: Any = getdot(caller, params)
+
+    # NOTE: check type of vars
+    if isinstance(getter, (str, int)):
+        return value.replace(found.group(0), str(getter))
+
+    # NOTE:
+    #   If type of getter caller does not formatting, it will return origin
+    #   value.
+    if value.replace(found.group(0), "") != "":
+        raise ValueError(
+            "Callable variable should not pass other outside ${{ ... }}"
+        )
+    return getter