ddeutil-workflow 0.0.1__tar.gz → 0.0.3__tar.gz

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.1
+Version: 0.0.3
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -23,15 +23,14 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
-Requires-Dist: ddeutil-model
 Requires-Dist: python-dotenv
 Provides-Extra: test
 Requires-Dist: sqlalchemy==2.0.30; extra == "test"
 Requires-Dist: paramiko==3.4.0; extra == "test"
 Requires-Dist: sshtunnel==0.4.0; extra == "test"
-Requires-Dist: boto3==1.34.109; extra == "test"
+Requires-Dist: boto3==1.34.117; extra == "test"
 Requires-Dist: fsspec==2024.5.0; extra == "test"
-Requires-Dist: polars==0.20.26; extra == "test"
+Requires-Dist: polars==0.20.31; extra == "test"
 Requires-Dist: pyarrow==16.1.0; extra == "test"
 
 # Data Utility: _Workflow_
@@ -83,6 +82,15 @@ The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
+The main feature of this project is the `Pipeline` object that can call any
+registries function. The pipeline can handle everything that you want to do, it
+will passing parameters and catching the output for re-use it to next step.
+
+> [!IMPORTANT]
+> In the future of this project, I will drop the connection and dataset to
+> dynamic registries instead of main features because it have a lot of maintain
+> vendor codes and deps. (I do not have time to handle this features)
+
 ### Connection
 
 The connection for worker able to do any thing.
@@ -102,7 +110,9 @@ assert conn.ping()
 
 ### Dataset
 
-The dataset is define any objects on the connection.
+The dataset is define any objects on the connection. This feature was implemented
+on `/vendors` because it has a lot of tools that can interact with any data systems
+in the data tool stacks.
 
 ```yaml
 ds_postgres_customer_tbl:
@@ -114,7 +124,7 @@ ds_postgres_customer_tbl:
 ```
 
 ```python
-from ddeutil.workflow.dataset import PostgresTbl
+from ddeutil.workflow.vendors.pg import PostgresTbl
 
 dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
 assert dataset.exists()
@@ -124,14 +134,14 @@ assert dataset.exists()
 
 ```yaml
 schd_for_node:
-  type: schedule.Scdl
+  type: schedule.Schedule
   cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.schedule import Scdl
+from ddeutil.workflow.schedule import Schedule
 
-scdl = Scdl.from_loader(name='schd_for_node', externals={})
+scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
 
 cron_iterate = scdl.generate('2022-01-01 00:00:00')
@@ -155,8 +165,10 @@ The state of doing lists that worker should to do. It be collection of the stage
 run_py_local:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    author-run: utils.receive.string
-    run-date: utils.receive.datetime
+    author-run:
+      type: str
+    run-date:
+      type: datetime
   jobs:
     first-job:
       stages:
@@ -203,13 +215,15 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 pipe_el_pg_to_lake:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run-date: utils.receive.datetime
-    author-email: utils.receive.string
+    run-date:
+      type: datetime
+    author-email:
+      type: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
-          id: extract
+          id: extract-load
           task: tasks/postgres-to-delta@polars
           with:
             source:
@@ -222,21 +236,22 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Hooks (Transform)
+### Tasks (Transform)
 
 ```yaml
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date: utils.receive.datetime
-    sp_name: utils.receive.string
-    source_name: utils.receive.string
-    target_name: utils.receive.string
+    run_date: datetime
+    sp_name: str
+    source_name: str
+    target_name: str
   jobs:
     transform:
       stages:
         - name: "Transform Data in MS SQL Server"
-          hook: hooks/mssql-proc@odbc
+          id: transform
+          task: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.3}/README.md

@@ -47,6 +47,15 @@ The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
+The main feature of this project is the `Pipeline` object that can call any
+registries function. The pipeline can handle everything that you want to do, it
+will passing parameters and catching the output for re-use it to next step.
+
+> [!IMPORTANT]
+> In the future of this project, I will drop the connection and dataset to
+> dynamic registries instead of main features because it have a lot of maintain
+> vendor codes and deps. (I do not have time to handle this features)
+
 ### Connection
 
 The connection for worker able to do any thing.
@@ -66,7 +75,9 @@ assert conn.ping()
 
 ### Dataset
 
-The dataset is define any objects on the connection.
+The dataset is define any objects on the connection. This feature was implemented
+on `/vendors` because it has a lot of tools that can interact with any data systems
+in the data tool stacks.
 
 ```yaml
 ds_postgres_customer_tbl:
@@ -78,7 +89,7 @@ ds_postgres_customer_tbl:
 ```
 
 ```python
-from ddeutil.workflow.dataset import PostgresTbl
+from ddeutil.workflow.vendors.pg import PostgresTbl
 
 dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
 assert dataset.exists()
@@ -88,14 +99,14 @@ assert dataset.exists()
 
 ```yaml
 schd_for_node:
-  type: schedule.Scdl
+  type: schedule.Schedule
   cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.schedule import Scdl
+from ddeutil.workflow.schedule import Schedule
 
-scdl = Scdl.from_loader(name='schd_for_node', externals={})
+scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
 
 cron_iterate = scdl.generate('2022-01-01 00:00:00')
@@ -119,8 +130,10 @@ The state of doing lists that worker should to do. It be collection of the stage
 run_py_local:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    author-run: utils.receive.string
-    run-date: utils.receive.datetime
+    author-run:
+      type: str
+    run-date:
+      type: datetime
   jobs:
     first-job:
       stages:
@@ -167,13 +180,15 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 pipe_el_pg_to_lake:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run-date: utils.receive.datetime
-    author-email: utils.receive.string
+    run-date:
+      type: datetime
+    author-email:
+      type: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
-          id: extract
+          id: extract-load
           task: tasks/postgres-to-delta@polars
           with:
             source:
@@ -186,21 +201,22 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Hooks (Transform)
+### Tasks (Transform)
 
 ```yaml
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date: utils.receive.datetime
-    sp_name: utils.receive.string
-    source_name: utils.receive.string
-    target_name: utils.receive.string
+    run_date: datetime
+    sp_name: str
+    source_name: str
+    target_name: str
   jobs:
     transform:
       stages:
         - name: "Transform Data in MS SQL Server"
-          hook: hooks/mssql-proc@odbc
+          id: transform
+          task: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.3}/pyproject.toml

@@ -28,7 +28,6 @@ requires-python = ">=3.9.13"
 dependencies = [
     "fmtutil",
     "ddeutil-io",
-    "ddeutil-model",
     "python-dotenv",
 ]
 dynamic = ["version"]
@@ -40,15 +39,15 @@ Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
 [project.optional-dependencies]
 test = [
     "sqlalchemy==2.0.30",
-    # SFTP warpper
+    # SFTP
     "paramiko==3.4.0",
     "sshtunnel==0.4.0",
-    # AWS
-    "boto3==1.34.109",
+    # AWS Client
+    "boto3==1.34.117",
     # Open files
     "fsspec==2024.5.0",
     # Polars
-    "polars==0.20.26",
+    "polars==0.20.31",
     "pyarrow==16.1.0",
 ]
 
@@ -80,6 +79,10 @@ addopts = [
     "--strict-markers",
 ]
 filterwarnings = ["error"]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 
 [tool.black]
 line-length = 80
@@ -98,7 +101,6 @@ exclude = """
         | build
         | dist
         | venv
-        | __legacy
     )/
 )
 """
@@ -114,7 +116,6 @@ exclude = [
     "build",
     "dist",
     "venv",
-    "__legacy",
 ]
 
 [tool.ruff.lint]
@@ -133,4 +134,7 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"__init__.py" = ["F401"]
+"__init__.py" = [
+    "F401",
+    "F403",
+]

ddeutil_workflow-0.0.3/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.3"

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.3}/src/ddeutil/workflow/__types.py

@@ -9,3 +9,4 @@ from typing import Any
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
+DictStr = dict[str, str]

{ddeutil_workflow-0.0.1 → ddeutil_workflow-0.0.3}/src/ddeutil/workflow/conn.py

@@ -10,7 +10,7 @@ from collections.abc import Iterator
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, TypeVar
 
-from ddeutil.model.conn import Conn as ConnModel
+from ddeutil.io.models.conn import Conn as ConnModel
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator
 from pydantic.types import SecretStr
@@ -43,27 +43,21 @@ class BaseConn(BaseModel):
     ]
 
     @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
-        """Construct Connection with Loader object with specific config name.
+    def from_dict(cls, values: DictData) -> Self:
+        """Construct Connection Model from dict data. This construct is
+        different with ``.model_validate()`` because it will prepare the values
+        before using it if the data dose not have 'url'.
 
-        :param name:
-        :param externals:
+        :param values: A dict data that use to construct this model.
         """
-        loader: Loader = Loader(name, externals=externals)
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        # NOTE: filter out the fields of this model.
         filter_data: DictData = {
-            k: loader.data.pop(k)
-            for k in loader.data.copy()
+            k: values.pop(k)
+            for k in values.copy()
             if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
         }
-        if "url" in loader.data:
-            url: ConnModel = ConnModel.from_url(loader.data.pop("url"))
+        if "url" in values:
+            url: ConnModel = ConnModel.from_url(values.pop("url"))
             return cls(
                 dialect=url.dialect,
                 host=url.host,
@@ -73,27 +67,38 @@ class BaseConn(BaseModel):
                 # NOTE:
                 #   I will replace None endpoint with memory value for SQLite
                 #   connection string.
-                endpoint=cls.__prepare_slash_from_url(url.endpoint or "memory"),
+                endpoint=(url.endpoint or "memory"),
                 # NOTE: This order will show that externals this the top level.
-                extras=(url.options | filter_data | externals),
+                extras=(url.options | filter_data),
             )
         return cls.model_validate(
             obj={
-                "extras": (
-                    loader.data.pop("extras", {}) | filter_data | externals
-                ),
-                **loader.data,
+                "extras": (values.pop("extras", {}) | filter_data),
+                **values,
             }
         )
 
     @classmethod
-    def __prepare_slash_from_url(cls, value: str) -> str:
-        if value.startswith("/"):
-            return value[1:]
-        return value
+    def from_loader(cls, name: str, externals: DictData) -> Self:
+        """Construct Connection with Loader object with specific config name.
+
+        :param name: A config name.
+        :param externals: A external data that want to adding to extras.
+        """
+        loader: Loader = Loader(name, externals=externals)
+        # NOTE: Validate the config type match with current connection model
+        if loader.type != cls:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        return cls.from_dict(
+            {
+                "extras": (loader.data.pop("extras", {}) | externals),
+                **loader.data,
+            }
+        )
 
     @field_validator("endpoint")
     def __prepare_slash(cls, value: str) -> str:
+        """Prepare slash character that map double form URL model loading."""
         if value.startswith("//"):
             return value[1:]
         return value
@@ -146,7 +151,7 @@ class SFTP(Conn):
     dialect: Literal["sftp"] = "sftp"
 
     def __client(self):
-        from .vendors.sftp_wrapped import WrapSFTP
+        from .vendors.sftp import WrapSFTP
 
         return WrapSFTP(
             host=self.host,

ddeutil_workflow-0.0.3/src/ddeutil/workflow/exceptions.py

@@ -0,0 +1,12 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""
+Define Errors Object for Node package
+"""
+from __future__ import annotations
+
+
+class TaskException(Exception): ...

ddeutil_workflow-0.0.3/src/ddeutil/workflow/loader.py

@@ -0,0 +1,174 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from functools import cached_property
+from typing import Any, ClassVar, TypeVar
+
+from ddeutil.core import (
+    getdot,
+    hasdot,
+    import_string,
+)
+from ddeutil.io import (
+    PathData,
+    PathSearch,
+    YamlEnvFl,
+)
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
+
+from .__regex import RegexConf
+from .__types import DictData
+
+T = TypeVar("T")
+BaseModelType = type[BaseModel]
+AnyModel = TypeVar("AnyModel", bound=BaseModel)
+
+
+class Engine(BaseModel):
+    """Engine Model"""
+
+    paths: PathData = Field(default_factory=PathData)
+    registry: list[str] = Field(default_factory=lambda: ["ddeutil.workflow"])
+
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        return values
+
+
+class Params(BaseModel):
+    """Params Model"""
+
+    engine: Engine = Field(default_factory=Engine)
+
+
+class SimLoad:
+    """Simple Load Object that will search config data by name.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param params: A Params model object.
+    :param externals: An external parameters
+
+    Noted:
+        The config data should have ``type`` key for engine can know what is
+    config should to do next.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        params: Params,
+        externals: DictData,
+    ) -> None:
+        self.data: DictData = {}
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in ("yml", "yaml")) and (
+                data := YamlEnvFl(file).read().get(name, {})
+            ):
+                self.data = data
+        if not self.data:
+            raise ValueError(f"Config {name!r} does not found on conf path")
+        self.__conf_params: Params = params
+        self.externals: DictData = externals
+
+    @property
+    def conf_params(self) -> Params:
+        return self.__conf_params
+
+    @cached_property
+    def type(self) -> BaseModelType:
+        """Return object type which implement in `config_object` key."""
+        if not (_typ := self.data.get("type")):
+            raise ValueError(
+                f"the 'type' value: {_typ} does not exists in config data."
+            )
+        try:
+            # NOTE: Auto adding module prefix if it does not set
+            return import_string(f"ddeutil.workflow.{_typ}")
+        except ModuleNotFoundError:
+            for registry in self.conf_params.engine.registry:
+                try:
+                    return import_string(f"{registry}.{_typ}")
+                except ModuleNotFoundError:
+                    continue
+            return import_string(f"{_typ}")
+
+    def load(self) -> AnyModel:
+        return self.type.model_validate(self.data)
+
+
+class Loader(SimLoad):
+    """Main Loader Object that get the config `yaml` file from current path.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param externals: An external parameters
+    """
+
+    conf_name: ClassVar[str] = "workflows-conf"
+
+    def __init__(
+        self,
+        name: str,
+        externals: DictData,
+        *,
+        path: str | None = None,
+    ) -> None:
+        self.data: DictData = {}
+
+        # NOTE: import params object from specific config file
+        params: Params = self.config(path)
+
+        super().__init__(name, params, externals)
+
+    @classmethod
+    def config(cls, path: str | None = None) -> Params:
+        """Load Config data from ``workflows-conf.yaml`` file."""
+        return Params.model_validate(
+            YamlEnvFl(path or f"./{cls.conf_name}.yaml").read()
+        )
+
+
+def map_params(value: Any, params: dict[str, Any]) -> Any:
+    """Map caller value that found from ``RE_CALLER`` regular expression.
+
+    :param value: A value that want to mapped with an params
+    :param params: A parameter value that getting with matched regular
+        expression.
+
+    :rtype: Any
+    :returns: An any getter value from the params input.
+    """
+    if isinstance(value, dict):
+        return {k: map_params(value[k], params) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([map_params(i, params) for i in value])
+    elif not isinstance(value, str):
+        return value
+
+    if not (found := RegexConf.RE_CALLER.search(value)):
+        return value
+
+    # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
+    caller: str = found.group("caller")
+    if not hasdot(caller, params):
+        raise ValueError(f"params does not set caller: {caller!r}")
+    getter: Any = getdot(caller, params)
+
+    # NOTE: check type of vars
+    if isinstance(getter, (str, int)):
+        return value.replace(found.group(0), str(getter))
+
+    # NOTE:
+    #   If type of getter caller does not formatting, it will return origin
+    #   value.
+    if value.replace(found.group(0), "") != "":
+        raise ValueError(
+            "Callable variable should not pass other outside ${{ ... }}"
+        )
+    return getter