ddeutil-workflow 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

ddeutil/workflow/utils.py

@@ -6,20 +6,105 @@
 from __future__ import annotations
 
 import inspect
+import os
 import stat
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from dataclasses import dataclass, field
 from datetime import date, datetime
 from functools import wraps
+from hashlib import md5
 from importlib import import_module
+from itertools import product
 from pathlib import Path
 from typing import Any, Callable, Literal, Optional, Protocol, Union
+from zoneinfo import ZoneInfo
 
-from ddeutil.core import lazy
+from ddeutil.core import getdot, hasdot, lazy
+from ddeutil.io import PathData
 from ddeutil.io.models.lineage import dt_now
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
+from .__types import DictData, Matrix, Re
+
+
+class Engine(BaseModel):
+    """Engine Model"""
+
+    paths: PathData = Field(default_factory=PathData)
+    registry: list[str] = Field(
+        default_factory=lambda: [
+            "ddeutil.workflow",
+        ],
+    )
+
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        """Prepare registry value that passing with string type. It convert the
+        string type to list of string.
+        """
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        return values
+
+
+class ConfParams(BaseModel):
+    """Params Model"""
+
+    engine: Engine = Field(
+        default_factory=Engine,
+        description="A engine mapping values.",
+    )
+
+
+def config() -> ConfParams:
+    """Load Config data from ``workflows-conf.yaml`` file."""
+    root_path: str = os.getenv("WORKFLOW_ROOT_PATH", ".")
+
+    regis: list[str] = []
+    if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
+        regis = [r.strip() for r in regis_env.split(",")]
+
+    conf_path: str = (
+        f"{root_path}/{conf_env}"
+        if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
+        else None
+    )
+    return ConfParams.model_validate(
+        obj={
+            "engine": {
+                "registry": regis,
+                "paths": {
+                    "root": root_path,
+                    "conf": conf_path,
+                },
+            },
+        }
+    )
+
+
+def gen_id(value: Any, *, sensitive: bool = True, unique: bool = False) -> str:
+    """Generate running ID for able to tracking. This generate process use `md5`
+    function.
+
+    :param value:
+    :param sensitive:
+    :param unique:
+    :rtype: str
+    """
+    if not isinstance(value, str):
+        value: str = str(value)
+
+    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
+    return md5(
+        (
+            f"{(value if sensitive else value.lower())}"
+            + (f"{datetime.now(tz=tz):%Y%m%d%H%M%S%f}" if unique else "")
+        ).encode()
+    ).hexdigest()
+
 
 class TagFunc(Protocol):
     """Tag Function Protocol"""
@@ -30,50 +115,68 @@ class TagFunc(Protocol):
     def __call__(self, *args, **kwargs): ...
 
 
-def tag(tag_value: str, name: str | None = None):
+def tag(value: str, name: str | None = None):
     """Tag decorator function that set function attributes, ``tag`` and ``name``
     for making registries variable.
 
-    :param: tag_value: A tag value for make different use-case of a function.
+    :param: value: A tag value for make different use-case of a function.
     :param: name: A name that keeping in registries.
     """
 
-    def func_internal(func: TagFunc):
-        func.tag = tag_value
+    def func_internal(func: callable) -> TagFunc:
+        func.tag = value
         func.name = name or func.__name__.replace("_", "-")
 
         @wraps(func)
         def wrapped(*args, **kwargs):
             return func(*args, **kwargs)
 
+        # TODO: pass result from a wrapped to Result model
+        #   >>> return Result.model_validate(obj=wrapped)
         return wrapped
 
     return func_internal
 
 
-def make_registry(module: str) -> dict[str, dict[str, Callable[[], TagFunc]]]:
-    """Return registries of all functions that able to called with task."""
-    rs: dict[str, dict[str, Callable[[], Callable]]] = {}
-    for fstr, func in inspect.getmembers(
-        import_module(module), inspect.isfunction
-    ):
-        if not hasattr(func, "tag"):
+Registry = dict[str, Callable[[], TagFunc]]
+
+
+def make_registry(submodule: str) -> dict[str, Registry]:
+    """Return registries of all functions that able to called with task.
+
+    :param submodule: A module prefix that want to import registry.
+    """
+    rs: dict[str, Registry] = {}
+    for module in config().engine.registry:
+        # NOTE: try to sequential import task functions
+        try:
+            importer = import_module(f"{module}.{submodule}")
+        except ModuleNotFoundError:
             continue
 
-        if func.name in rs:
+        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
+            # NOTE: check function attribute that already set tag by
+            #   ``utils.tag`` decorator.
+            if not hasattr(func, "tag"):
+                continue
+
+            # NOTE: Create new register name if it not exists
+            if func.name not in rs:
+                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
+                continue
+
             if func.tag in rs[func.name]:
                 raise ValueError(
-                    f"The tag {func.tag!r} already exists on module {module}"
+                    f"The tag {func.tag!r} already exists on "
+                    f"{module}.{submodule}, you should change this tag name or "
+                    f"change it func name."
                 )
-            rs[func.name][func.tag] = lazy(f"{module}.{fstr}")
-            continue
+            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
 
-        # NOTE: Create new register name if it not exists
-        rs[func.name] = {func.tag: lazy(f"{module}.{fstr}")}
     return rs
 
 
-class BaseParams(BaseModel, ABC):
+class BaseParam(BaseModel, ABC):
     """Base Parameter that use to make Params Model."""
 
     desc: Optional[str] = None
@@ -87,7 +190,7 @@ class BaseParams(BaseModel, ABC):
         )
 
 
-class DefaultParams(BaseParams):
+class DefaultParam(BaseParam):
     """Default Parameter that will check default if it required"""
 
     default: Optional[str] = None
@@ -107,7 +210,7 @@ class DefaultParams(BaseParams):
         return self
 
 
-class DatetimeParams(DefaultParams):
+class DatetimeParam(DefaultParam):
     """Datetime parameter."""
 
     type: Literal["datetime"] = "datetime"
@@ -130,7 +233,7 @@ class DatetimeParams(DefaultParams):
         return datetime.fromisoformat(value)
 
 
-class StrParams(DefaultParams):
+class StrParam(DefaultParam):
     """String parameter."""
 
     type: Literal["str"] = "str"
@@ -141,7 +244,7 @@ class StrParams(DefaultParams):
         return str(value)
 
 
-class IntParams(DefaultParams):
+class IntParam(DefaultParam):
     """Integer parameter."""
 
     type: Literal["int"] = "int"
@@ -160,7 +263,7 @@ class IntParams(DefaultParams):
         return value
 
 
-class ChoiceParams(BaseParams):
+class ChoiceParam(BaseParam):
     type: Literal["choice"] = "choice"
     options: list[str]
 
@@ -175,13 +278,101 @@ class ChoiceParams(BaseParams):
         return value
 
 
-Params = Union[
-    ChoiceParams,
-    DatetimeParams,
-    StrParams,
+Param = Union[
+    ChoiceParam,
+    DatetimeParam,
+    StrParam,
 ]
 
 
+@dataclass
+class Result:
+    """Result Dataclass object for passing parameter and receiving output from
+    the pipeline execution.
+    """
+
+    status: int = field(default=2)
+    context: DictData = field(default_factory=dict)
+
+
 def make_exec(path: str | Path):
+    """Change mode of file to be executable file."""
     f: Path = Path(path) if isinstance(path, str) else path
     f.chmod(f.stat().st_mode | stat.S_IEXEC)
+
+
+def param2template(
+    value: Any,
+    params: dict[str, Any],
+    *,
+    repr_flag: bool = False,
+) -> Any:
+    """Pass param to template string that can search by ``RE_CALLER`` regular
+    expression.
+
+    :param value: A value that want to mapped with an params
+    :param params: A parameter value that getting with matched regular
+        expression.
+    :param repr_flag: A repr flag for using repr instead of str if it set be
+        true.
+
+    :rtype: Any
+    :returns: An any getter value from the params input.
+    """
+    if isinstance(value, dict):
+        return {k: param2template(value[k], params) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([param2template(i, params) for i in value])
+    elif not isinstance(value, str):
+        return value
+
+    if not Re.RE_CALLER.search(value):
+        return value
+
+    for found in Re.RE_CALLER.finditer(value):
+
+        # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
+        caller: str = found.group("caller")
+        if not hasdot(caller, params):
+            raise ValueError(f"params does not set caller: {caller!r}")
+
+        getter: Any = getdot(caller, params)
+
+        # NOTE: check type of vars
+        if isinstance(getter, (str, int)):
+            value: str = value.replace(
+                found.group(0), (repr(getter) if repr_flag else str(getter)), 1
+            )
+            continue
+
+        # NOTE:
+        #   If type of getter caller does not formatting, it will return origin
+        #   value from the ``getdot`` function.
+        if value.replace(found.group(0), "", 1) != "":
+            raise ValueError(
+                "Callable variable should not pass other outside ${{ ... }}"
+            )
+        return getter
+    return value
+
+
+def dash2underscore(
+    key: str,
+    values: DictData,
+    *,
+    fixed: str | None = None,
+) -> DictData:
+    """Change key name that has dash to underscore."""
+    if key in values:
+        values[(fixed or key.replace("-", "_"))] = values.pop(key)
+    return values
+
+
+def cross_product(matrix: Matrix) -> Iterator:
+    """Iterator of products value from matrix."""
+    yield from (
+        {_k: _v for e in mapped for _k, _v in e.items()}
+        for mapped in product(
+            *[[{k: v} for v in vs] for k, vs in matrix.items()]
+        )
+    )

{ddeutil_workflow-0.0.4.dist-info → ddeutil_workflow-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.4
+Version: 0.0.6
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -9,7 +9,7 @@ Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
 Keywords: data,workflow,utility,pipeline
 Classifier: Topic :: Utilities
 Classifier: Natural Language :: English
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -23,35 +23,33 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
-Requires-Dist: python-dotenv
-Provides-Extra: test
-Requires-Dist: sqlalchemy ==2.0.30 ; extra == 'test'
-Requires-Dist: paramiko ==3.4.0 ; extra == 'test'
-Requires-Dist: sshtunnel ==0.4.0 ; extra == 'test'
-Requires-Dist: boto3 ==1.34.117 ; extra == 'test'
-Requires-Dist: fsspec ==2024.5.0 ; extra == 'test'
-Requires-Dist: polars ==0.20.31 ; extra == 'test'
-Requires-Dist: pyarrow ==16.1.0 ; extra == 'test'
-
-# Data Utility: _Workflow_
+Requires-Dist: python-dotenv ==1.0.1
+Provides-Extra: app
+Requires-Dist: fastapi ==0.112.0 ; extra == 'app'
+Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
+
+# Workflow
 
 [![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
 [![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
 [![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
+[![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
 
 **Table of Contents**:
 
 - [Installation](#installation)
 - [Getting Started](#getting-started)
-  - [Connection](#connection)
-  - [Dataset](#dataset)
-  - [Schedule](#schedule)
-- [Examples](#examples)
-  - [Python](#python)
-  - [Tasks (EL)](#tasks-extract--load)
-  - [Hooks (T)](#hooks-transform)
-
-This **Utility Workflow** objects was created for easy to make a simple metadata
+- [Core Features](#core-features)
+  - [On](#on)
+  - [Pipeline](#pipeline)
+- [Usage](#usage)
+  - [Python & Bash](#python--bash)
+  - [Hook (EL)](#hook-extract--load)
+  - [Hook (T)](#hook-transform)
+- [Configuration](#configuration)
+- [Deployment](#deployment)
+
+This **Workflow** objects was created for easy to make a simple metadata
 driven pipeline that able to **ETL, T, EL, or ELT** by `.yaml` file.
 
 I think we should not create the multiple pipeline per use-case if we able to
@@ -74,13 +72,18 @@ pipeline.
 pip install ddeutil-workflow
 ```
 
-This project need `ddeutil-io`, `ddeutil-model` extension namespace packages.
+This project need `ddeutil-io` extension namespace packages. If you want to install
+this package with application add-ons, you should add `app` in installation;
+
+```shell
+pip install ddeutil-workflow[app]
+```
 
 ## Getting Started
 
 The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
-is similar component of the **Airflow** because I like it concepts.
+is similar component of the **Airflow** because I like it orchestration concepts.
 
 The main feature of this project is the `Pipeline` object that can call any
 registries function. The pipeline can handle everything that you want to do, it
@@ -91,88 +94,84 @@ will passing parameters and catching the output for re-use it to next step.
 > dynamic registries instead of main features because it have a lot of maintain
 > vendor codes and deps. (I do not have time to handle this features)
 
-### Connection
+### On
 
-The connection for worker able to do any thing.
+The **On** is schedule object.
 
 ```yaml
-conn_postgres_data:
-  type: conn.Postgres
-  url: 'postgres//username:${ENV_PASS}@hostname:port/database?echo=True&time_out=10'
+on_every_5_min:
+  type: on.On
+  cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.conn import Conn
+from ddeutil.workflow.on import On
 
-conn = Conn.from_loader(name='conn_postgres_data', externals={})
-assert conn.ping()
-```
+schedule = On.from_loader(name='on_every_5_min', externals={})
+assert '*/5 * * * *' == str(schedule.cronjob)
 
-### Dataset
-
-The dataset is define any objects on the connection. This feature was implemented
-on `/vendors` because it has a lot of tools that can interact with any data systems
-in the data tool stacks.
-
-```yaml
-ds_postgres_customer_tbl:
-  type: dataset.PostgresTbl
-  conn: 'conn_postgres_data'
-  features:
-    id: serial primary key
-    name: varchar( 100 ) not null
+cron_iter = schedule.generate('2022-01-01 00:00:00')
+assert '2022-01-01 00:05:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:10:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:15:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:20:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
 ```
 
-```python
-from ddeutil.workflow.vendors.pg import PostgresTbl
-
-dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
-assert dataset.exists()
-```
+### Pipeline
 
-### Schedule
+The **Pipeline** object that is the core feature of this project.
 
 ```yaml
-schd_for_node:
-  type: schedule.Schedule
-  cron: "*/5 * * * *"
+run_py_local:
+  type: ddeutil.workflow.pipeline.Pipeline
+  on: 'on_every_5_min'
+  params:
+    author-run:
+      type: str
+    run-date:
+      type: datetime
 ```
 
 ```python
-from ddeutil.workflow.schedule import Schedule
-
-scdl = Schedule.from_loader(name='schd_for_node', externals={})
-assert '*/5 * * * *' == str(scdl.cronjob)
+from ddeutil.workflow.pipeline import Pipeline
 
-cron_iterate = scdl.generate('2022-01-01 00:00:00')
-assert '2022-01-01 00:05:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:10:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:15:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:20:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:25:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+pipe = Pipeline.from_loader(name='run_py_local', externals={})
+pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 ```
 
-## Examples
+> [!NOTE]
+> The above parameter use short declarative statement. You can pass a parameter
+> type to the key of a parameter name.
+> ```yaml
+> params:
+>   author-run: str
+>   run-date: datetime
+> ```
+>
+> And for the type, you can remove `ddeutil.workflow` prefix because we can find
+> it by looping search from `WORKFLOW_CORE_REGISTRY` value.
+
+## Usage
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
 
-### Python
+> [!IMPORTANT]
+> I recommend you to use `task` stage for all actions that you want to do with
+> pipeline object.
 
-The state of doing lists that worker should to do. It be collection of the stage.
+### Python & Bash
 
 ```yaml
 run_py_local:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: pipeline.Pipeline
   params:
-    author-run:
-      type: str
-    run-date:
-      type: datetime
+    author-run: str
+    run-date: datetime
   jobs:
     first-job:
       stages:
-        - name: Printing Information
+        - name: "Printing Information"
           id: define-func
           run: |
             x = '${{ params.author-run }}'
@@ -181,7 +180,7 @@ run_py_local:
             def echo(name: str):
               print(f'Hello {name}')
 
-        - name: Run Sequence and use var from Above
+        - name: "Run Sequence and use var from Above"
           vars:
             x: ${{ params.author-run }}
           run: |
@@ -189,11 +188,17 @@ run_py_local:
             # Change x value
             x: int = 1
 
-        - name: Call Function
+        - name: "Call Function"
           vars:
             echo: ${{ stages.define-func.outputs.echo }}
           run: |
             echo('Caller')
+    second-job:
+      stages:
+        - name: "Echo Bash Script"
+          id: shell-echo
+          bash: |
+            echo "Hello World from Shell"
 ```
 
 ```python
@@ -207,24 +212,23 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 > Hello Local Workflow
 > Receive x from above with Local Workflow
 > Hello Caller
+> Hello World from Shell
 ```
 
-### Tasks (Extract & Load)
+### Hook (Extract & Load)
 
 ```yaml
 pipe_el_pg_to_lake:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: pipeline.Pipeline
   params:
-    run-date:
-      type: datetime
-    author-email:
-      type: str
+    run-date: datetime
+    author-email: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
           id: extract-load
-          task: tasks/postgres-to-delta@polars
+          uses: tasks/postgres-to-delta@polars
           with:
             source:
               conn: conn_postgres_url
@@ -236,11 +240,11 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Tasks (Transform)
+### Hook (Transform)
 
 ```yaml
-pipe_hook_mssql_proc:
-  type: ddeutil.workflow.pipe.Pipeline
+pipeline_hook_mssql_proc:
+  type: pipeline.Pipeline
   params:
     run_date: datetime
     sp_name: str
@@ -251,7 +255,7 @@ pipe_hook_mssql_proc:
       stages:
         - name: "Transform Data in MS SQL Server"
           id: transform
-          task: tasks/mssql-proc@odbc
+          uses: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:
@@ -261,6 +265,30 @@ pipe_hook_mssql_proc:
               target: ${{ params.target_name }}
 ```
 
-## License
+## Configuration
 
-This project was licensed under the terms of the [MIT license](LICENSE).
+```bash
+export WORKFLOW_ROOT_PATH=.
+export WORKFLOW_CORE_REGISTRY=ddeutil.workflow,tests.utils
+export WORKFLOW_CORE_PATH_CONF=conf
+```
+
+Application config:
+
+```bash
+export WORKFLOW_APP_DB_URL=postgresql+asyncpg://user:pass@localhost:5432/schedule
+export WORKFLOW_APP_INTERVAL=10
+```
+
+## Deployment
+
+This package able to run as a application service for receive manual trigger
+from the master node via RestAPI.
+
+> [!WARNING]
+> This feature do not start yet because I still research and find the best tool
+> to use it provision an app service, like `starlette`, `fastapi`, `apscheduler`.
+
+```shell
+(venv) $ workflow start -p 7070
+```