ddeutil-workflow 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

ddeutil/workflow/utils.py

@@ -0,0 +1,65 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import inspect
+from functools import wraps
+from importlib import import_module
+from typing import Callable, Protocol
+
+from ddeutil.core import lazy
+
+
+class TagFunc(Protocol):
+    """Tag Function Protocol"""
+
+    name: str
+    tag: str
+
+    def __call__(self, *args, **kwargs): ...
+
+
+def tag(tag_value: str, name: str | None = None):
+    """Tag decorator function that set function attributes, ``tag`` and ``name``
+    for making registries variable.
+
+    :param: tag_value: A tag value for make different use-case of a function.
+    :param: name: A name that keeping in registries.
+    """
+
+    def func_internal(func: TagFunc):
+        func.tag = tag_value
+        func.name = name or func.__name__.replace("_", "-")
+
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    return func_internal
+
+
+def make_registry(module: str) -> dict[str, dict[str, Callable[[], TagFunc]]]:
+    """Return registries of all functions that able to called with task."""
+    rs: dict[str, dict[str, Callable[[], Callable]]] = {}
+    for fstr, func in inspect.getmembers(
+        import_module(module), inspect.isfunction
+    ):
+        if not hasattr(func, "tag"):
+            continue
+
+        if func.name in rs:
+            if func.tag in rs[func.name]:
+                raise ValueError(
+                    f"The tag {func.tag!r} already exists on module {module}"
+                )
+            rs[func.name][func.tag] = lazy(f"{module}.{fstr}")
+            continue
+
+        # NOTE: Create new register name if it not exists
+        rs[func.name] = {func.tag: lazy(f"{module}.{fstr}")}
+    return rs

{ddeutil_workflow-0.0.1.dist-info → ddeutil_workflow-0.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.1
+Version: 0.0.2
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -23,15 +23,14 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
-Requires-Dist: ddeutil-model
 Requires-Dist: python-dotenv
 Provides-Extra: test
 Requires-Dist: sqlalchemy ==2.0.30 ; extra == 'test'
 Requires-Dist: paramiko ==3.4.0 ; extra == 'test'
 Requires-Dist: sshtunnel ==0.4.0 ; extra == 'test'
-Requires-Dist: boto3 ==1.34.109 ; extra == 'test'
+Requires-Dist: boto3 ==1.34.117 ; extra == 'test'
 Requires-Dist: fsspec ==2024.5.0 ; extra == 'test'
-Requires-Dist: polars ==0.20.26 ; extra == 'test'
+Requires-Dist: polars ==0.20.31 ; extra == 'test'
 Requires-Dist: pyarrow ==16.1.0 ; extra == 'test'
 
 # Data Utility: _Workflow_
@@ -83,6 +82,9 @@ The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
 is similar component of the **Airflow** because I like it concepts.
 
+The main feature of this project is the `Pipeline` object that can call any
+registried function.
+
 ### Connection
 
 The connection for worker able to do any thing.
@@ -155,8 +157,10 @@ The state of doing lists that worker should to do. It be collection of the stage
 run_py_local:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    author-run: utils.receive.string
-    run-date: utils.receive.datetime
+    author-run:
+      type: str
+    run-date:
+      type: datetime
   jobs:
     first-job:
       stages:
@@ -203,13 +207,15 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 pipe_el_pg_to_lake:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run-date: utils.receive.datetime
-    author-email: utils.receive.string
+    run-date:
+      type: datetime
+    author-email:
+      type: str
   jobs:
     extract-load:
       stages:
         - name: "Extract Load from Postgres to Lake"
-          id: extract
+          id: extract-load
           task: tasks/postgres-to-delta@polars
           with:
             source:
@@ -222,21 +228,26 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
-### Hooks (Transform)
+### Tasks (Transform)
 
 ```yaml
 pipe_hook_mssql_proc:
   type: ddeutil.workflow.pipe.Pipeline
   params:
-    run_date: utils.receive.datetime
-    sp_name: utils.receive.string
-    source_name: utils.receive.string
-    target_name: utils.receive.string
+    run_date:
+      type: datetime
+    sp_name:
+      type: str
+    source_name:
+      type: str
+    target_name:
+      type: str
   jobs:
     transform:
       stages:
         - name: "Transform Data in MS SQL Server"
-          hook: hooks/mssql-proc@odbc
+          id: transform
+          task: tasks/mssql-proc@odbc
           with:
             exec: ${{ params.sp_name }}
             params:

ddeutil_workflow-0.0.2.dist-info/RECORD

@@ -0,0 +1,25 @@
+ddeutil/workflow/__about__.py,sha256=Ow9Rd-50zm5qXBM0iKjjMszFJAKgfR8LsocpgUKYVrI,27
+ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
+ddeutil/workflow/__types.py,sha256=7t-i5_-eif9zUBwb7jnv5RbSjzOihyl3yFDqXzaeyxk,428
+ddeutil/workflow/conn.py,sha256=KpQywXVurVvUjOdDg8zrPp1mKyRwR6qqrEjkPBhi_yc,6838
+ddeutil/workflow/dataset.py,sha256=Qszg65YyMCy6Bojya_OSbXtUjCqS8sZnDJvJtu706YA,8578
+ddeutil/workflow/exceptions.py,sha256=LAa3QlnPqECGxq6PpXoXNZVca7v7l4szjTZ3zWXJoWM,824
+ddeutil/workflow/loader.py,sha256=iTE1Rhjl-bq3YHZCKYWY51iwbQie2N6-MsaduOF1qSg,4463
+ddeutil/workflow/pipeline.py,sha256=r3FjbiC51eDArkoOQmfoSE--bZgggxUvRUh99bj3A9Y,14529
+ddeutil/workflow/schedule.py,sha256=f3V6dM66fM2C67vt9ltFSyqmxCMx1yo_gJuUDR_RMIE,2672
+ddeutil/workflow/utils.py,sha256=j2CmadYq9nXTyGWYGp_L0LS-fOnkTyzTNpf359ne1dA,2011
+ddeutil/workflow/tasks/__init__.py,sha256=TIcw9JinrdepWgyazSMLk_QflUFms99ILI4GvLHUGD0,338
+ddeutil/workflow/tasks/_pandas.py,sha256=ob8Ozq9ASiky8SwxWcR8HUr_j-qQoJez6EEckHrfR3s,1693
+ddeutil/workflow/tasks/_polars.py,sha256=0EbASTqc-aNtNnLN1QIYONqb5gzJvoqqZ-V584UNQtI,2708
+ddeutil/workflow/vendors/__dict.py,sha256=ETwkeA0qzKNgedfeRgAz1qShNXTIXIS4DXzJB4lM4jo,9962
+ddeutil/workflow/vendors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/vendors/__schedule.py,sha256=cUIwtTli9G0klrNFxAIbG3VTiv6umRuNAZiKA-kOtpE,20690
+ddeutil/workflow/vendors/aws_warpped.py,sha256=zjq_LCu3ffVBRrxS2vqss9X24yrtuAEt9ouy2_WvS0o,5980
+ddeutil/workflow/vendors/minio_warpped.py,sha256=pScLy38Du9moOrGaSBSFsoQRhiQ686FQyloOeLA0OQk,261
+ddeutil/workflow/vendors/sftp_wrapped.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
+ddeutil_workflow-0.0.2.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.2.dist-info/METADATA,sha256=o1ECtqH1WSBpz0UNu9OdzFqmxrmpbZP2H-KqHF4l8i0,7511
+ddeutil_workflow-0.0.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+ddeutil_workflow-0.0.2.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.2.dist-info/RECORD,,

ddeutil/workflow/hooks/__init__.py

@@ -1,9 +0,0 @@
-from typing import Any
-
-from ddeutil.core import lazy
-
-registries: dict[str, Any] = {
-    "postgres-proc": {
-        "pysycopg": lazy("ddeutil.workflow.tasks._postgres.postgres_procedure"),
-    },
-}

ddeutil/workflow/hooks/_postgres.py

@@ -1,2 +0,0 @@
-def postgres_procedure():
-    return

ddeutil/workflow/utils/receive.py

@@ -1,33 +0,0 @@
-import datetime as dt
-from typing import Any
-
-
-def datetime(value: Any) -> dt.datetime:
-    if isinstance(value, dt.datetime):
-        return value
-    elif isinstance(value, dt.date):
-        return dt.datetime(value.year, value.month, value.day)
-    if value is None:
-        return dt.datetime.now(dt.timezone.utc)
-    elif not isinstance(value, str):
-        raise ValueError(
-            f"Value that want to convert to datetime does not support for "
-            f"type: {type(value)}"
-        )
-    return dt.datetime.fromisoformat(value)
-
-
-def string(value: Any) -> str:
-    return str(value)
-
-
-def integer(value: Any) -> int:
-    if not isinstance(value, int):
-        try:
-            return int(str(value))
-        except TypeError as err:
-            raise ValueError(
-                f"Value that want to convert to integer does not support for "
-                f"type: {type(value)}"
-            ) from err
-    return value

ddeutil/workflow/utils/selection.py

@@ -1,2 +0,0 @@
-def conn(value):
-    return value

ddeutil_workflow-0.0.1.dist-info/RECORD

@@ -1,28 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=YZoDMFgikTmgL5EqRKa4mX-zXFr2c20J7dSZJ4dqSd0,27
-ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
-ddeutil/workflow/__types.py,sha256=7t-i5_-eif9zUBwb7jnv5RbSjzOihyl3yFDqXzaeyxk,428
-ddeutil/workflow/conn.py,sha256=5Nf6IQf2PXC6eLn469tGNvtnXAx-eOm3jJ_ltj90VbQ,6831
-ddeutil/workflow/dataset.py,sha256=SNp58WwTvgDSFWll9aizJeiHX_fd68f687cRME7vBo4,8599
-ddeutil/workflow/exceptions.py,sha256=aV_LN4_zmAk2R2p5ZYJcSMrsdBsRo6cMwm0lwZv_400,2399
-ddeutil/workflow/loader.py,sha256=nJffdJ8kchhwYsEGnWoG_leR16dzU0evo_p5Wbi4rp4,9710
-ddeutil/workflow/pipeline.py,sha256=kN6k2ht4C2OPHZYPljpnUQqfY3K54kMSOS1azGt7R68,10755
-ddeutil/workflow/schedule.py,sha256=RSvT0pi1Gb1un3OseRUW9AngRgVuAlXFT23QrSjtxYk,2828
-ddeutil/workflow/hooks/__init__.py,sha256=x5pY43037-cE0j3ofU_f7oEylzx-sN7pQwhiZubBtvY,199
-ddeutil/workflow/hooks/_postgres.py,sha256=8bAAlxHe_K1sAqwE1Pz6y6cKuytimmPnaEeudEPag2A,37
-ddeutil/workflow/tasks/__init__.py,sha256=NgGzbxWBV6lRgIEvp4VuREbKzo_ncTLgx00W2BAR-uk,276
-ddeutil/workflow/tasks/_polars.py,sha256=OT6kt7mYHx88DEJsO8sfnIfcjdXZ02XCAAcQBKr6wXM,1429
-ddeutil/workflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/utils/receive.py,sha256=lWB8k-_hPmdkq2bjPSL_2IX1X19I8f017u8-7jNRH68,939
-ddeutil/workflow/utils/selection.py,sha256=0qJu4mgVfNLLlnBbrsQtKK__LmG_DZf2Gs8CBplP4c0,34
-ddeutil/workflow/vendors/__dict.py,sha256=ETwkeA0qzKNgedfeRgAz1qShNXTIXIS4DXzJB4lM4jo,9962
-ddeutil/workflow/vendors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/vendors/__schedule.py,sha256=cUIwtTli9G0klrNFxAIbG3VTiv6umRuNAZiKA-kOtpE,20690
-ddeutil/workflow/vendors/aws_warpped.py,sha256=zjq_LCu3ffVBRrxS2vqss9X24yrtuAEt9ouy2_WvS0o,5980
-ddeutil/workflow/vendors/minio_warpped.py,sha256=pScLy38Du9moOrGaSBSFsoQRhiQ686FQyloOeLA0OQk,261
-ddeutil/workflow/vendors/sftp_wrapped.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
-ddeutil_workflow-0.0.1.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.1.dist-info/METADATA,sha256=uFH3MTuZk6UNPjLIQqaQSebszywROMUMzjLMftY0OM0,7444
-ddeutil_workflow-0.0.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ddeutil_workflow-0.0.1.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.1.dist-info/RECORD,,