ddeutil-workflow 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

ddeutil/workflow/utils.py

@@ -6,22 +6,104 @@
 from __future__ import annotations
 
 import inspect
+import os
 import stat
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from dataclasses import dataclass, field
 from datetime import date, datetime
 from functools import wraps
+from hashlib import md5
 from importlib import import_module
+from itertools import product
 from pathlib import Path
 from typing import Any, Callable, Literal, Optional, Protocol, Union
+from zoneinfo import ZoneInfo
 
-import msgspec as spec
-from ddeutil.core import lazy
+from ddeutil.core import getdot, hasdot, lazy
+from ddeutil.io import PathData
 from ddeutil.io.models.lineage import dt_now
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData
+from .__types import DictData, Matrix, Re
+
+
+class Engine(BaseModel):
+    """Engine Model"""
+
+    paths: PathData = Field(default_factory=PathData)
+    registry: list[str] = Field(
+        default_factory=lambda: [
+            "ddeutil.workflow",
+        ],
+    )
+
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        """Prepare registry value that passing with string type. It convert the
+        string type to list of string.
+        """
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        return values
+
+
+class ConfParams(BaseModel):
+    """Params Model"""
+
+    engine: Engine = Field(
+        default_factory=Engine,
+        description="A engine mapping values.",
+    )
+
+
+def config() -> ConfParams:
+    """Load Config data from ``workflows-conf.yaml`` file."""
+    root_path: str = os.getenv("WORKFLOW_ROOT_PATH", ".")
+
+    regis: list[str] = []
+    if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
+        regis = [r.strip() for r in regis_env.split(",")]
+
+    conf_path: str = (
+        f"{root_path}/{conf_env}"
+        if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
+        else None
+    )
+    return ConfParams.model_validate(
+        obj={
+            "engine": {
+                "registry": regis,
+                "paths": {
+                    "root": root_path,
+                    "conf": conf_path,
+                },
+            },
+        }
+    )
+
+
+def gen_id(value: Any, *, sensitive: bool = True, unique: bool = False) -> str:
+    """Generate running ID for able to tracking. This generate process use `md5`
+    function.
+
+    :param value:
+    :param sensitive:
+    :param unique:
+    :rtype: str
+    """
+    if not isinstance(value, str):
+        value: str = str(value)
+
+    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
+    return md5(
+        (
+            f"{(value if sensitive else value.lower())}"
+            + (f"{datetime.now(tz=tz):%Y%m%d%H%M%S%f}" if unique else "")
+        ).encode()
+    ).hexdigest()
 
 
 class TagFunc(Protocol):
@@ -49,37 +131,52 @@ def tag(value: str, name: str | None = None):
         def wrapped(*args, **kwargs):
             return func(*args, **kwargs)
 
+        # TODO: pass result from a wrapped to Result model
+        #   >>> return Result.model_validate(obj=wrapped)
         return wrapped
 
     return func_internal
 
 
-def make_registry(module: str) -> dict[str, dict[str, Callable[[], TagFunc]]]:
+Registry = dict[str, Callable[[], TagFunc]]
+
+
+def make_registry(submodule: str) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
 
-    :param module: A module prefix that want to import registry.
+    :param submodule: A module prefix that want to import registry.
     """
-    rs: dict[str, dict[str, Callable[[], Callable]]] = {}
-    for fstr, func in inspect.getmembers(
-        import_module(module), inspect.isfunction
-    ):
-        if not hasattr(func, "tag"):
+    rs: dict[str, Registry] = {}
+    for module in config().engine.registry:
+        # NOTE: try to sequential import task functions
+        try:
+            importer = import_module(f"{module}.{submodule}")
+        except ModuleNotFoundError:
             continue
 
-        if func.name in rs:
+        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
+            # NOTE: check function attribute that already set tag by
+            #   ``utils.tag`` decorator.
+            if not hasattr(func, "tag"):
+                continue
+
+            # NOTE: Create new register name if it not exists
+            if func.name not in rs:
+                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
+                continue
+
             if func.tag in rs[func.name]:
                 raise ValueError(
-                    f"The tag {func.tag!r} already exists on module {module}"
+                    f"The tag {func.tag!r} already exists on "
+                    f"{module}.{submodule}, you should change this tag name or "
+                    f"change it func name."
                 )
-            rs[func.name][func.tag] = lazy(f"{module}.{fstr}")
-            continue
+            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
 
-        # NOTE: Create new register name if it not exists
-        rs[func.name] = {func.tag: lazy(f"{module}.{fstr}")}
     return rs
 
 
-class BaseParams(BaseModel, ABC):
+class BaseParam(BaseModel, ABC):
     """Base Parameter that use to make Params Model."""
 
     desc: Optional[str] = None
@@ -93,7 +190,7 @@ class BaseParams(BaseModel, ABC):
         )
 
 
-class DefaultParams(BaseParams):
+class DefaultParam(BaseParam):
     """Default Parameter that will check default if it required"""
 
     default: Optional[str] = None
@@ -113,7 +210,7 @@ class DefaultParams(BaseParams):
         return self
 
 
-class DatetimeParams(DefaultParams):
+class DatetimeParam(DefaultParam):
     """Datetime parameter."""
 
     type: Literal["datetime"] = "datetime"
@@ -136,7 +233,7 @@ class DatetimeParams(DefaultParams):
         return datetime.fromisoformat(value)
 
 
-class StrParams(DefaultParams):
+class StrParam(DefaultParam):
     """String parameter."""
 
     type: Literal["str"] = "str"
@@ -147,7 +244,7 @@ class StrParams(DefaultParams):
         return str(value)
 
 
-class IntParams(DefaultParams):
+class IntParam(DefaultParam):
     """Integer parameter."""
 
     type: Literal["int"] = "int"
@@ -166,7 +263,7 @@ class IntParams(DefaultParams):
         return value
 
 
-class ChoiceParams(BaseParams):
+class ChoiceParam(BaseParam):
     type: Literal["choice"] = "choice"
     options: list[str]
 
@@ -181,28 +278,101 @@ class ChoiceParams(BaseParams):
         return value
 
 
-Params = Union[
-    ChoiceParams,
-    DatetimeParams,
-    StrParams,
+Param = Union[
+    ChoiceParam,
+    DatetimeParam,
+    StrParam,
 ]
 
 
+@dataclass
+class Result:
+    """Result Dataclass object for passing parameter and receiving output from
+    the pipeline execution.
+    """
+
+    status: int = field(default=2)
+    context: DictData = field(default_factory=dict)
+
+
 def make_exec(path: str | Path):
     """Change mode of file to be executable file."""
     f: Path = Path(path) if isinstance(path, str) else path
     f.chmod(f.stat().st_mode | stat.S_IEXEC)
 
 
-class TaskSearch(spec.Struct, kw_only=True, tag="task"):
-    """Task Search Struct that use the `msgspec` for the best performance data
-    serialize.
+def param2template(
+    value: Any,
+    params: dict[str, Any],
+    *,
+    repr_flag: bool = False,
+) -> Any:
+    """Pass param to template string that can search by ``RE_CALLER`` regular
+    expression.
+
+    :param value: A value that want to mapped with an params
+    :param params: A parameter value that getting with matched regular
+        expression.
+    :param repr_flag: A repr flag for using repr instead of str if it set be
+        true.
+
+    :rtype: Any
+    :returns: An any getter value from the params input.
     """
+    if isinstance(value, dict):
+        return {k: param2template(value[k], params) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([param2template(i, params) for i in value])
+    elif not isinstance(value, str):
+        return value
 
-    path: str
-    func: str
-    tag: str
+    if not Re.RE_CALLER.search(value):
+        return value
+
+    for found in Re.RE_CALLER.finditer(value):
 
-    def to_dict(self) -> DictData:
-        """Return dict data from struct fields."""
-        return {f: getattr(self, f) for f in self.__struct_fields__}
+        # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
+        caller: str = found.group("caller")
+        if not hasdot(caller, params):
+            raise ValueError(f"params does not set caller: {caller!r}")
+
+        getter: Any = getdot(caller, params)
+
+        # NOTE: check type of vars
+        if isinstance(getter, (str, int)):
+            value: str = value.replace(
+                found.group(0), (repr(getter) if repr_flag else str(getter)), 1
+            )
+            continue
+
+        # NOTE:
+        #   If type of getter caller does not formatting, it will return origin
+        #   value from the ``getdot`` function.
+        if value.replace(found.group(0), "", 1) != "":
+            raise ValueError(
+                "Callable variable should not pass other outside ${{ ... }}"
+            )
+        return getter
+    return value
+
+
+def dash2underscore(
+    key: str,
+    values: DictData,
+    *,
+    fixed: str | None = None,
+) -> DictData:
+    """Change key name that has dash to underscore."""
+    if key in values:
+        values[(fixed or key.replace("-", "_"))] = values.pop(key)
+    return values
+
+
+def cross_product(matrix: Matrix) -> Iterator:
+    """Iterator of products value from matrix."""
+    yield from (
+        {_k: _v for e in mapped for _k, _v in e.items()}
+        for mapped in product(
+            *[[{k: v} for v in vs] for k, vs in matrix.items()]
+        )
+    )

{ddeutil_workflow-0.0.5.dist-info → ddeutil_workflow-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.5
+Version: 0.0.6
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,30 +24,32 @@ License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
 Requires-Dist: python-dotenv ==1.0.1
-Requires-Dist: schedule ==1.2.2
+Provides-Extra: app
+Requires-Dist: fastapi ==0.112.0 ; extra == 'app'
+Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
 
-# Data Utility: _Workflow_
+# Workflow
 
 [![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
 [![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
 [![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
 [![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
 
-
 **Table of Contents**:
 
 - [Installation](#installation)
 - [Getting Started](#getting-started)
-  - [Connection](#connection)
-  - [Dataset](#dataset)
-  - [Schedule](#schedule)
-- [Pipeline Examples](#examples)
-  - [Python & Shell](#python--shell)
-  - [Tasks (EL)](#tasks-extract--load)
-  - [Hooks (T)](#tasks-transform)
+- [Core Features](#core-features)
+  - [On](#on)
+  - [Pipeline](#pipeline)
+- [Usage](#usage)
+  - [Python & Bash](#python--bash)
+  - [Hook (EL)](#hook-extract--load)
+  - [Hook (T)](#hook-transform)
 - [Configuration](#configuration)
+- [Deployment](#deployment)
 
-This **Utility Workflow** objects was created for easy to make a simple metadata
+This **Workflow** objects was created for easy to make a simple metadata
 driven pipeline that able to **ETL, T, EL, or ELT** by `.yaml` file.
 
 I think we should not create the multiple pipeline per use-case if we able to
@@ -70,7 +72,12 @@ pipeline.
 pip install ddeutil-workflow
 ```
 
-This project need `ddeutil-io`, `ddeutil-model` extension namespace packages.
+This project need `ddeutil-io` extension namespace packages. If you want to install
+this package with application add-ons, you should add `app` in installation;
+
+```shell
+pip install ddeutil-workflow[app]
+```
 
 ## Getting Started
 
@@ -87,38 +94,42 @@ will passing parameters and catching the output for re-use it to next step.
 > dynamic registries instead of main features because it have a lot of maintain
 > vendor codes and deps. (I do not have time to handle this features)
 
----
+### On
 
-### Schedule
+The **On** is schedule object.
 
 ```yaml
-schd_for_node:
-  type: schedule.Schedule
+on_every_5_min:
+  type: on.On
   cron: "*/5 * * * *"
 ```
 
 ```python
-from ddeutil.workflow.on import Schedule
+from ddeutil.workflow.on import On
 
-scdl = Schedule.from_loader(name='schd_for_node', externals={})
-assert '*/5 * * * *' == str(scdl.cronjob)
+schedule = On.from_loader(name='on_every_5_min', externals={})
+assert '*/5 * * * *' == str(schedule.cronjob)
 
-cron_iterate = scdl.generate('2022-01-01 00:00:00')
-assert '2022-01-01 00:05:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:10:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:15:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:20:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:25:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+cron_iter = schedule.generate('2022-01-01 00:00:00')
+assert '2022-01-01 00:05:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:10:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:15:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:20:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
 ```
 
----
-
 ### Pipeline
 
+The **Pipeline** object that is the core feature of this project.
+
 ```yaml
 run_py_local:
   type: ddeutil.workflow.pipeline.Pipeline
-  ...
+  on: 'on_every_5_min'
+  params:
+    author-run:
+      type: str
+    run-date:
+      type: datetime
 ```
 
 ```python
@@ -128,27 +139,39 @@ pipe = Pipeline.from_loader(name='run_py_local', externals={})
 pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 ```
 
-## Examples
+> [!NOTE]
+> The above parameter use short declarative statement. You can pass a parameter
+> type to the key of a parameter name.
+> ```yaml
+> params:
+>   author-run: str
+>   run-date: datetime
+> ```
+>
+> And for the type, you can remove `ddeutil.workflow` prefix because we can find
+> it by looping search from `WORKFLOW_CORE_REGISTRY` value.
+
+## Usage
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
 
-### Python & Shell
+> [!IMPORTANT]
+> I recommend you to use `task` stage for all actions that you want to do with
+> pipeline object.
 
-The state of doing lists that worker should to do. It be collection of the stage.
+### Python & Bash
 
 ```yaml
 run_py_local:
-  type: ddeutil.workflow.pipeline.Pipeline
+  type: pipeline.Pipeline
   params:
-    author-run:
-      type: str
-    run-date:
-      type: datetime
+    author-run: str
+    run-date: datetime
   jobs:
     first-job:
       stages:
-        - name: Printing Information
+        - name: "Printing Information"
           id: define-func
           run: |
             x = '${{ params.author-run }}'
@@ -157,7 +180,7 @@ run_py_local:
             def echo(name: str):
               print(f'Hello {name}')
 
-        - name: Run Sequence and use var from Above
+        - name: "Run Sequence and use var from Above"
           vars:
             x: ${{ params.author-run }}
           run: |
@@ -165,16 +188,16 @@ run_py_local:
             # Change x value
             x: int = 1
 
-        - name: Call Function
+        - name: "Call Function"
           vars:
             echo: ${{ stages.define-func.outputs.echo }}
           run: |
             echo('Caller')
     second-job:
       stages:
-        - name: Echo Shell Script
+        - name: "Echo Bash Script"
           id: shell-echo
-          shell: |
+          bash: |
             echo "Hello World from Shell"
 ```
 
@@ -192,24 +215,20 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 > Hello World from Shell
 ```
 
----
-
-### Tasks (Extract & Load)
+### Hook (Extract & Load)
 
 ```yaml
 pipe_el_pg_to_lake:
-  type: ddeutil.workflow.pipeline.Pipeline
+  type: pipeline.Pipeline
   params:
-    run-date:
-      type: datetime
-    author-email:
-      type: str
+    run-date: datetime
+    author-email: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
           id: extract-load
-          task: tasks/postgres-to-delta@polars
+          uses: tasks/postgres-to-delta@polars
           with:
             source:
               conn: conn_postgres_url
@@ -221,15 +240,11 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
----
-
-### Tasks (Transform)
-
-> I recommend you to use task for all actions that you want to do.
+### Hook (Transform)
 
 ```yaml
-pipe_hook_mssql_proc:
-  type: ddeutil.workflow.pipeline.Pipeline
+pipeline_hook_mssql_proc:
+  type: pipeline.Pipeline
   params:
     run_date: datetime
     sp_name: str
@@ -240,7 +255,7 @@ pipe_hook_mssql_proc:
       stages:
         - name: "Transform Data in MS SQL Server"
           id: transform
-          task: tasks/mssql-proc@odbc
+          uses: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:
@@ -250,16 +265,30 @@ pipe_hook_mssql_proc:
               target: ${{ params.target_name }}
 ```
 
-> [!NOTE]
-> The above parameter use short declarative statement. You can pass a parameter
-> type to the key of a parameter name.
-
 ## Configuration
 
-```text
+```bash
+export WORKFLOW_ROOT_PATH=.
+export WORKFLOW_CORE_REGISTRY=ddeutil.workflow,tests.utils
+export WORKFLOW_CORE_PATH_CONF=conf
+```
+
+Application config:
 
+```bash
+export WORKFLOW_APP_DB_URL=postgresql+asyncpg://user:pass@localhost:5432/schedule
+export WORKFLOW_APP_INTERVAL=10
 ```
 
-## License
+## Deployment
 
-This project was licensed under the terms of the [MIT license](LICENSE).
+This package able to run as a application service for receive manual trigger
+from the master node via RestAPI.
+
+> [!WARNING]
+> This feature do not start yet because I still research and find the best tool
+> to use it provision an app service, like `starlette`, `fastapi`, `apscheduler`.
+
+```shell
+(venv) $ workflow start -p 7070
+```

ddeutil_workflow-0.0.6.dist-info/RECORD

@@ -0,0 +1,15 @@
+ddeutil/workflow/__about__.py,sha256=VEYa91VchyTUnF57lvvquHvitTViBxxLXuhcEnr4TAY,27
+ddeutil/workflow/__init__.py,sha256=Y5wLiJ0zS1CfoSOZ0oo7OL3LNMKvPmpUO4fVHuAOv8E,429
+ddeutil/workflow/__types.py,sha256=PfwDZBnTwe2JImD7UFS0J6Nq-1TcjBGebOVzJZoSuTQ,1354
+ddeutil/workflow/exceptions.py,sha256=9O12c4aNLi0dyjVBgCLveV3HN9PXcZfwFfLXdgm3Ffs,626
+ddeutil/workflow/loader.py,sha256=_ZD-XP5P7VbUeqItrUVPaKIZu6dMUZ2aywbCbReW1hQ,2778
+ddeutil/workflow/on.py,sha256=YoEqDbzJUwqOA3JRltbvlYr0rNTtxdmb7cWMxl8U19k,6717
+ddeutil/workflow/pipeline.py,sha256=8mIvY34_fsiqscBa9JB94MgN3Km5fkuD2iaNZSAQVuM,17843
+ddeutil/workflow/scheduler.py,sha256=FqmkvWCqwJ4eRf8aDn5Ce4FcNWqmcvu2aTTfL34lfgs,22184
+ddeutil/workflow/stage.py,sha256=bDJiGS21gYlYbFDnLTKH9aIbXfej9fT-V1ADoPX7w4s,13829
+ddeutil/workflow/utils.py,sha256=xapKxxnqIzlbKA45GaRcWn-VL30AhE7M8f46ynr-vbI,11173
+ddeutil_workflow-0.0.6.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.6.dist-info/METADATA,sha256=5X6ewXGn96MR9rDhVmmaoTuIwuRmfi72t7ezM8wxbvw,8612
+ddeutil_workflow-0.0.6.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+ddeutil_workflow-0.0.6.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.6.dist-info/RECORD,,

ddeutil/workflow/__regex.py

@@ -1,44 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# --------------------------------------------------------------------------
-import re
-from re import (
-    IGNORECASE,
-    MULTILINE,
-    UNICODE,
-    VERBOSE,
-    Pattern,
-)
-
-
-class RegexConf:
-    """Regular expression config."""
-
-    # NOTE: Search caller
-    __re_caller: str = r"""
-        \$
-        {{
-            \s*(?P<caller>
-                [a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?
-            )\s*
-        }}
-    """
-    RE_CALLER: Pattern = re.compile(
-        __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
-    )
-
-    # NOTE: Search task
-    __re_task_fmt: str = r"""
-        ^
-            (?P<path>[^/@]+)
-            /
-            (?P<func>[^@]+)
-            @
-            (?P<tag>.+)
-        $
-    """
-    RE_TASK_FMT: Pattern = re.compile(
-        __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
-    )